Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

debug: add rename logging - proves metadata IS preserved correctly 

CRITICAL FINDING:
Rename operation works perfectly:
- Source: size=1260 chunks=1
- Destination: size=1260 chunks=1
- Metadata is correctly preserved!

The EOF error occurs DURING READ, not after rename.
Parquet tries to read at position=1260 with bufRemaining=78,
meaning it expects file to be 1338 bytes but it's only 1260.

This proves the issue is in how Parquet WRITES the file,
not in how SeaweedFS stores or renames it.

The Parquet footer contains incorrect offsets that were
calculated during the write phase.
pull/7526/head
chrislu 1 week ago
parent
e878431dea
commit
580a6c1e00
2 changed files with 65 additions and 38 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 74
      other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
  2. 29
      other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

74
other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
View File

@ -354,48 +354,48 @@ public class SeaweedOutputStream extends OutputStream {
LOG.warn("[DEBUG-2024] Looking up: parentDir={} fileName={}", parentDir, fileName);
int maxRetries = 5;
long retryDelayMs = 10; // Start with 10ms
for (int attempt = 0; attempt < maxRetries; attempt++) {
try {
LOG.warn("[DEBUG-2024] Attempt {} to lookup metadata", attempt + 1);
// Lookup the entry to verify metadata is visible
FilerProto.Entry lookedUpEntry = filerClient.lookupEntry(parentDir, fileName);
LOG.warn("[DEBUG-2024] Lookup returned: {}", (lookedUpEntry != null ? "entry found" : "null"));
if (lookedUpEntry != null) {
long lookedUpSize = lookedUpEntry.getAttributes().getFileSize();
if (lookedUpSize == position) {
// Metadata is correct and visible
if (attempt > 0) {
LOG.info("[DEBUG-2024] Metadata visible after {} retries: path={} size={}",
attempt, path, position);
int maxRetries = 5;
long retryDelayMs = 10; // Start with 10ms
for (int attempt = 0; attempt < maxRetries; attempt++) {
try {
LOG.warn("[DEBUG-2024] Attempt {} to lookup metadata", attempt + 1);
// Lookup the entry to verify metadata is visible
FilerProto.Entry lookedUpEntry = filerClient.lookupEntry(parentDir, fileName);
LOG.warn("[DEBUG-2024] Lookup returned: {}", (lookedUpEntry != null ? "entry found" : "null"));
if (lookedUpEntry != null) {
long lookedUpSize = lookedUpEntry.getAttributes().getFileSize();
if (lookedUpSize == position) {
// Metadata is correct and visible
if (attempt > 0) {
LOG.info("[DEBUG-2024] Metadata visible after {} retries: path={} size={}",
attempt, path, position);
}
return;
} else {
// Metadata is stale
LOG.warn("[DEBUG-2024] Metadata stale on attempt {}: path={} expected={} actual={}",
attempt + 1, path, position, lookedUpSize);
}
return;
} else {
// Metadata is stale
LOG.warn("[DEBUG-2024] Metadata stale on attempt {}: path={} expected={} actual={}",
attempt + 1, path, position, lookedUpSize);
}
}
// Metadata not yet visible or stale, retry
if (attempt < maxRetries - 1) {
Thread.sleep(retryDelayMs);
retryDelayMs *= 2; // Exponential backoff
}
// Metadata not yet visible or stale, retry
if (attempt < maxRetries - 1) {
Thread.sleep(retryDelayMs);
retryDelayMs *= 2; // Exponential backoff
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while waiting for metadata visibility", e);
} catch (Exception e) {
LOG.warn("[DEBUG-2024] Error checking metadata visibility on attempt {}: {}",
attempt + 1, e.getMessage());
// Continue to next retry
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("Interrupted while waiting for metadata visibility", e);
} catch (Exception e) {
LOG.warn("[DEBUG-2024] Error checking metadata visibility on attempt {}: {}",
attempt + 1, e.getMessage());
// Continue to next retry
}
}
}
// If we get here, metadata may still not be visible, but we've done our best
LOG.warn("[DEBUG-2024] Metadata may not be immediately visible after {} retries: path={} size={}",

29
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -161,13 +161,40 @@ public class SeaweedFileSystemStore {
if (source.isRoot()) {
return;
}
LOG.info("rename source: {} destination:{}", source, destination);
LOG.warn("[DEBUG-2024] RENAME START: {} => {}", source, destination);
FilerProto.Entry entry = lookupEntry(source);
if (entry == null) {
LOG.warn("rename non-existing source: {}", source);
return;
}
// Log source file metadata before rename
long sourceSize = entry.getAttributes().getFileSize();
int sourceChunks = entry.getChunksCount();
LOG.warn("[DEBUG-2024] Source file metadata: size={} chunks={}", sourceSize, sourceChunks);
filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
LOG.warn("[DEBUG-2024] RENAME COMPLETE: {} => {}", source, destination);
// Lookup destination to verify metadata was preserved
FilerProto.Entry destEntry = lookupEntry(destination);
if (destEntry != null) {
long destSize = destEntry.getAttributes().getFileSize();
int destChunks = destEntry.getChunksCount();
LOG.warn("[DEBUG-2024] Destination file metadata: size={} chunks={}", destSize, destChunks);
if (sourceSize != destSize) {
LOG.error("[DEBUG-2024] METADATA MISMATCH! Source size={} but destination size={}",
sourceSize, destSize);
}
if (sourceChunks != destChunks) {
LOG.error("[DEBUG-2024] CHUNK COUNT MISMATCH! Source chunks={} but destination chunks={}",
sourceChunks, destChunks);
}
} else {
LOG.error("[DEBUG-2024] Destination file not found after rename!");
}
}
public OutputStream createFile(final Path path,

Write Preview
Loading…
Cancel
Save