Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Honor collection vif for EC shard config

rust-volume-server
Chris Lu 3 days ago
parent
9a7eb93b8a
commit
e299726d7d
2 changed files with 68 additions and 26 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 31
      seaweed-volume/src/server/grpc_server.rs
  2. 63
      seaweed-volume/src/storage/erasure_coding/ec_volume.rs

31
seaweed-volume/src/server/grpc_server.rs
View File

@ -1733,25 +1733,10 @@ impl VolumeServer for VolumeGrpcService {
};
// Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo)
let (data_shards, parity_shards) = {
let base = crate::storage::volume::volume_file_name(&dir, collection, vid);
let vif_path = format!("{}.vif", base);
let mut ds = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
let mut ps = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
if let Ok(vif_info) =
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
{
if let Some(ec) = vif_info.ec_shard_config {
if ec.data_shards > 0 && ec.parity_shards > 0 {
ds = ec.data_shards;
ps = ec.parity_shards;
}
}
}
}
(ds, ps)
};
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir, collection, vid,
);
crate::storage::erasure_coding::ec_encoder::write_ec_files(
&dir,
@ -1814,7 +1799,9 @@ impl VolumeServer for VolumeGrpcService {
// Check which shards are missing
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(&dir, vid);
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir, collection, vid,
);
let total_shards = data_shards + parity_shards;
let mut missing: Vec<u32> = Vec::new();
@ -2954,7 +2941,9 @@ impl VolumeServer for VolumeGrpcService {
total_volumes += 1;
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(&dir, *vid);
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir, &collection, *vid,
);
match crate::storage::erasure_coding::ec_encoder::verify_ec_shards(
&dir,

63
seaweed-volume/src/storage/erasure_coding/ec_volume.rs
View File

@ -35,16 +35,20 @@ pub struct EcVolume {
pub expire_at_sec: u64,
}
pub fn read_ec_shard_config(dir: &str, volume_id: VolumeId) -> (u32, u32) {
pub fn read_ec_shard_config(dir: &str, collection: &str, volume_id: VolumeId) -> (u32, u32) {
let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
let vif_path = format!("{}/{}.vif", dir, volume_id.0);
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let vif_path = format!("{}.vif", base);
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
if let Ok(vif_info) =
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
{
if let Some(ec) = vif_info.ec_shard_config {
if ec.data_shards > 0 && ec.parity_shards > 0 {
if ec.data_shards > 0
&& ec.parity_shards > 0
&& (ec.data_shards + ec.parity_shards) <= TOTAL_SHARDS_COUNT as u32
{
data_shards = ec.data_shards;
parity_shards = ec.parity_shards;
}
@ -62,7 +66,7 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
let (data_shards, parity_shards) = read_ec_shard_config(dir, volume_id);
let (data_shards, parity_shards) = read_ec_shard_config(dir, collection, volume_id);
let total_shards = (data_shards + parity_shards) as usize;
let mut shards = Vec::with_capacity(total_shards);
@ -425,7 +429,6 @@ impl EcVolume {
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::idx;
use tempfile::TempDir;
fn write_ecx_file(
@ -510,4 +513,54 @@ mod tests {
assert_eq!(vol.shard_count(), 1);
assert!(vol.shard_bits().has_shard_id(3));
}
#[test]
fn test_ec_volume_uses_collection_prefixed_vif_config() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
write_ecx_file(dir, "pics", VolumeId(1), &[]);
let vif = crate::storage::volume::VifVolumeInfo {
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 6,
parity_shards: 3,
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.data_shards, 6);
assert_eq!(vol.parity_shards, 3);
}
#[test]
fn test_ec_volume_invalid_vif_config_falls_back_to_defaults() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
write_ecx_file(dir, "pics", VolumeId(1), &[]);
let vif = crate::storage::volume::VifVolumeInfo {
ec_shard_config: Some(crate::storage::volume::VifEcShardConfig {
data_shards: 10,
parity_shards: 10,
}),
..Default::default()
};
let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
std::fs::write(
format!("{}.vif", base),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.data_shards, DATA_SHARDS_COUNT as u32);
assert_eq!(vol.parity_shards, PARITY_SHARDS_COUNT as u32);
}
}
Write Preview
Loading…
Cancel
Save