Browse Source

ec: fall back to data dir when ecx file not found in idx dir

When -dir.idx is configured after EC encoding, .ecx/.ecj files remain
in the data directory. Port Go PR #8541 logic:

- EcVolume::new() falls back to data dir for .ecx/.ecj, tracks actual
  location in ecx_actual_dir field
- load_existing_volumes() checks both idx and data dirs for .ecx
- remove_ec_volume_files() cleans up .ecx/.ecj from both directories
- delete_ec_shards() removes .ecx/.ecj from both idx and data dirs
- find_ec_dir() checks idx dir first, falls back to data dir
- destroy() removes .ecx/.ecj from all possible locations
rust-volume-server
Chris Lu 5 days ago
parent
commit
596c85fdd8
  1. 18
      seaweed-volume/src/storage/disk_location.rs
  2. 46
      seaweed-volume/src/storage/erasure_coding/ec_volume.rs
  3. 30
      seaweed-volume/src/storage/store.rs

18
seaweed-volume/src/storage/disk_location.rs

@ -136,7 +136,16 @@ impl DiskLocation {
// If valid EC shards exist (.ecx file present), skip loading .dat // If valid EC shards exist (.ecx file present), skip loading .dat
let ecx_path = format!("{}.ecx", idx_name); let ecx_path = format!("{}.ecx", idx_name);
if std::path::Path::new(&ecx_path).exists() {
let ecx_exists = if std::path::Path::new(&ecx_path).exists() {
true
} else if self.idx_directory != self.directory {
// .ecx may have been created before -dir.idx was configured
let fallback = format!("{}.ecx", volume_name);
std::path::Path::new(&fallback).exists()
} else {
false
};
if ecx_exists {
if self.validate_ec_volume(&collection, vid) { if self.validate_ec_volume(&collection, vid) {
// Valid EC volume — don't load .dat // Valid EC volume — don't load .dat
continue; continue;
@ -288,9 +297,14 @@ impl DiskLocation {
let idx_base = volume_file_name(&self.idx_directory, collection, vid); let idx_base = volume_file_name(&self.idx_directory, collection, vid);
const MAX_SHARD_COUNT: usize = 32; const MAX_SHARD_COUNT: usize = 32;
// Remove index files first (.ecx, .ecj)
// Remove index files from idx directory (.ecx, .ecj)
let _ = fs::remove_file(format!("{}.ecx", idx_base)); let _ = fs::remove_file(format!("{}.ecx", idx_base));
let _ = fs::remove_file(format!("{}.ecj", idx_base)); let _ = fs::remove_file(format!("{}.ecj", idx_base));
// Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
if self.idx_directory != self.directory {
let _ = fs::remove_file(format!("{}.ecx", base));
let _ = fs::remove_file(format!("{}.ecj", base));
}
// Remove all EC shard files (.ec00 ~ .ec31) // Remove all EC shard files (.ec00 ~ .ec31)
for i in 0..MAX_SHARD_COUNT { for i in 0..MAX_SHARD_COUNT {

46
seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@ -26,6 +26,8 @@ pub struct EcVolume {
ecx_file_size: i64, ecx_file_size: i64,
ecj_file: Option<File>, ecj_file: Option<File>,
pub disk_type: DiskType, pub disk_type: DiskType,
/// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
ecx_actual_dir: String,
/// Maps shard ID -> list of server addresses where that shard exists. /// Maps shard ID -> list of server addresses where that shard exists.
/// Used for distributed EC reads across the cluster. /// Used for distributed EC reads across the cluster.
pub shard_locations: HashMap<ShardId, Vec<String>>, pub shard_locations: HashMap<ShardId, Vec<String>>,
@ -100,20 +102,38 @@ impl EcVolume {
ecx_file_size: 0, ecx_file_size: 0,
ecj_file: None, ecj_file: None,
disk_type: DiskType::default(), disk_type: DiskType::default(),
ecx_actual_dir: dir_idx.to_string(),
shard_locations: HashMap::new(), shard_locations: HashMap::new(),
expire_at_sec, expire_at_sec,
}; };
// Open .ecx file (sorted index)
// Open .ecx file (sorted index), with fallback to data dir
let ecx_path = vol.ecx_file_name(); let ecx_path = vol.ecx_file_name();
if std::path::Path::new(&ecx_path).exists() { if std::path::Path::new(&ecx_path).exists() {
let file = File::open(&ecx_path)?; let file = File::open(&ecx_path)?;
vol.ecx_file_size = file.metadata()?.len() as i64; vol.ecx_file_size = file.metadata()?.len() as i64;
vol.ecx_file = Some(file); vol.ecx_file = Some(file);
} else if dir_idx != dir {
// Fall back to data directory if .ecx was created before -dir.idx was configured
let data_base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let fallback_ecx = format!("{}.ecx", data_base);
if std::path::Path::new(&fallback_ecx).exists() {
tracing::info!(
volume_id = volume_id.0,
"ecx file not found in idx dir, falling back to data dir"
);
let file = File::open(&fallback_ecx)?;
vol.ecx_file_size = file.metadata()?.len() as i64;
vol.ecx_file = Some(file);
vol.ecx_actual_dir = dir.to_string();
}
} }
// Open .ecj file (deletion journal)
let ecj_path = vol.ecj_file_name();
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
let ecj_base = crate::storage::volume::volume_file_name(
&vol.ecx_actual_dir, collection, volume_id,
);
let ecj_path = format!("{}.ecj", ecj_base);
let ecj_file = OpenOptions::new() let ecj_file = OpenOptions::new()
.read(true) .read(true)
.write(true) .write(true)
@ -377,8 +397,24 @@ impl EcVolume {
} }
*shard = None; *shard = None;
} }
let _ = fs::remove_file(self.ecx_file_name());
let _ = fs::remove_file(self.ecj_file_name());
// Remove .ecx/.ecj from ecx_actual_dir (where they were found)
let actual_base = crate::storage::volume::volume_file_name(
&self.ecx_actual_dir, &self.collection, self.volume_id,
);
let _ = fs::remove_file(format!("{}.ecx", actual_base));
let _ = fs::remove_file(format!("{}.ecj", actual_base));
// Also try the configured idx dir and data dir in case files exist in either
if self.ecx_actual_dir != self.dir_idx {
let _ = fs::remove_file(self.ecx_file_name());
let _ = fs::remove_file(self.ecj_file_name());
}
if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
let data_base = crate::storage::volume::volume_file_name(
&self.dir, &self.collection, self.volume_id,
);
let _ = fs::remove_file(format!("{}.ecx", data_base));
let _ = fs::remove_file(format!("{}.ecj", data_base));
}
self.ecx_file = None; self.ecx_file = None;
self.ecj_file = None; self.ecj_file = None;
} }

30
seaweed-volume/src/storage/store.rs

@ -550,14 +550,21 @@ impl Store {
// Also unmount if mounted // Also unmount if mounted
self.unmount_ec_shards(vid, shard_ids); self.unmount_ec_shards(vid, shard_ids);
// If all shards are gone, remove .ecx and .ecj files
// If all shards are gone, remove .ecx and .ecj files from both idx and data dirs
let all_gone = self.check_all_ec_shards_deleted(vid, collection); let all_gone = self.check_all_ec_shards_deleted(vid, collection);
if all_gone { if all_gone {
for loc in &self.locations { for loc in &self.locations {
let base =
crate::storage::volume::volume_file_name(&loc.directory, collection, vid);
let _ = std::fs::remove_file(format!("{}.ecx", base));
let _ = std::fs::remove_file(format!("{}.ecj", base));
let idx_base =
crate::storage::volume::volume_file_name(&loc.idx_directory, collection, vid);
let _ = std::fs::remove_file(format!("{}.ecx", idx_base));
let _ = std::fs::remove_file(format!("{}.ecj", idx_base));
// Also try data directory in case .ecx/.ecj were created before -dir.idx
if loc.idx_directory != loc.directory {
let data_base =
crate::storage::volume::volume_file_name(&loc.directory, collection, vid);
let _ = std::fs::remove_file(format!("{}.ecx", data_base));
let _ = std::fs::remove_file(format!("{}.ecj", data_base));
}
} }
} }
} }
@ -578,11 +585,18 @@ impl Store {
/// Find the directory containing EC files for a volume. /// Find the directory containing EC files for a volume.
pub fn find_ec_dir(&self, vid: VolumeId, collection: &str) -> Option<String> { pub fn find_ec_dir(&self, vid: VolumeId, collection: &str) -> Option<String> {
for loc in &self.locations { for loc in &self.locations {
let base = crate::storage::volume::volume_file_name(&loc.directory, collection, vid);
let ecx_path = format!("{}.ecx", base);
if std::path::Path::new(&ecx_path).exists() {
// Check idx directory first
let idx_base = crate::storage::volume::volume_file_name(&loc.idx_directory, collection, vid);
if std::path::Path::new(&format!("{}.ecx", idx_base)).exists() {
return Some(loc.directory.clone()); return Some(loc.directory.clone());
} }
// Fall back to data directory if .ecx was created before -dir.idx was configured
if loc.idx_directory != loc.directory {
let data_base = crate::storage::volume::volume_file_name(&loc.directory, collection, vid);
if std::path::Path::new(&format!("{}.ecx", data_base)).exists() {
return Some(loc.directory.clone());
}
}
} }
None None
} }

Loading…
Cancel
Save