Browse Source

fix: EC encode reads .idx from separate index directory

write_ec_files now accepts an idx_dir parameter so it can find the .idx
file when --dir.idx is configured. Previously it looked for .idx in the
data directory, which failed when a separate index directory was in use.
rust-volume-server
Chris Lu 9 hours ago
parent
commit
57dde4492c
  1. 4
      seaweed-volume/src/server/grpc_server.rs
  2. 2
      seaweed-volume/src/storage/erasure_coding/ec_decoder.rs
  3. 6
      seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

4
seaweed-volume/src/server/grpc_server.rs

@ -1649,7 +1649,7 @@ impl VolumeServer for VolumeGrpcService {
let collection = &req.collection;
// Find the volume's directory and validate collection
let (dir, vol_version, dat_file_size) = {
let (dir, idx_dir, vol_version, dat_file_size) = {
let store = self.state.store.read().unwrap();
let (loc_idx, vol) = store
.find_volume(vid)
@ -1664,6 +1664,7 @@ impl VolumeServer for VolumeGrpcService {
let dat_size = vol.dat_file_size().unwrap_or(0) as i64;
(
store.locations[loc_idx].directory.clone(),
store.locations[loc_idx].idx_directory.clone(),
version,
dat_size,
)
@ -1692,6 +1693,7 @@ impl VolumeServer for VolumeGrpcService {
crate::storage::erasure_coding::ec_encoder::write_ec_files(
&dir,
&idx_dir,
collection,
vid,
data_shards as usize,

2
seaweed-volume/src/storage/erasure_coding/ec_decoder.rs

@ -216,7 +216,7 @@ mod tests {
// Encode to EC
let data_shards = 10;
let parity_shards = 4;
ec_encoder::write_ec_files(dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
ec_encoder::write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
// Delete original .dat and .idx
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();

6
seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@ -21,6 +21,7 @@ use crate::storage::volume::volume_file_name;
/// Also creates a sorted .ecx index from the .idx file.
pub fn write_ec_files(
dir: &str,
idx_dir: &str,
collection: &str,
volume_id: VolumeId,
data_shards: usize,
@ -28,7 +29,8 @@ pub fn write_ec_files(
) -> io::Result<()> {
let base = volume_file_name(dir, collection, volume_id);
let dat_path = format!("{}.dat", base);
let idx_path = format!("{}.idx", base);
let idx_base = volume_file_name(idx_dir, collection, volume_id);
let idx_path = format!("{}.idx", idx_base);
// Create sorted .ecx from .idx
write_sorted_ecx_from_idx(&idx_path, &format!("{}.ecx", base))?;
@ -481,7 +483,7 @@ mod tests {
let data_shards = 10;
let parity_shards = 4;
let total_shards = data_shards + parity_shards;
write_ec_files(dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
// Verify shard files exist
for i in 0..total_shards {

Loading…
Cancel
Save