Browse Source

debug: add aggressive logging to FSDataOutputStream getPos() override

This will help determine:
1. If the anonymous FSDataOutputStream subclass is being created
2. If the getPos() override is actually being called by Parquet
3. What position value is being returned

If we see 'Creating FSDataOutputStream' but NOT 'getPos() override called',
it means FSDataOutputStream is using a different mechanism for position tracking.

If we don't see either log, it means the code path isn't being used at all.
pull/7526/head
chrislu 1 week ago
parent
commit
d6f9234cea
  1. 12
      other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

12
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@ -114,11 +114,13 @@ public class SeaweedFileSystem extends FileSystem {
SeaweedHadoopOutputStream outputStream = (SeaweedHadoopOutputStream) seaweedFileSystemStore.createFile(path, overwrite, permission,
seaweedBufferSize, replicaPlacement);
// Use custom FSDataOutputStream that delegates getPos() to our stream
LOG.info("[DEBUG-2024] Creating FSDataOutputStream with custom getPos() override for path: {}", path);
return new FSDataOutputStream(outputStream, statistics) {
@Override
public long getPos() {
// Delegate to SeaweedOutputStream's position tracking
return outputStream.getPos();
long pos = outputStream.getPos();
LOG.info("[DEBUG-2024] FSDataOutputStream.getPos() override called! Returning: {} for path: {}", pos, path);
return pos;
}
};
} catch (Exception ex) {
@ -164,11 +166,13 @@ public class SeaweedFileSystem extends FileSystem {
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
SeaweedHadoopOutputStream outputStream = (SeaweedHadoopOutputStream) seaweedFileSystemStore.createFile(path, false, null, seaweedBufferSize, "");
// Use custom FSDataOutputStream that delegates getPos() to our stream
LOG.info("[DEBUG-2024] Creating FSDataOutputStream (append) with custom getPos() override for path: {}", path);
return new FSDataOutputStream(outputStream, statistics) {
@Override
public long getPos() {
// Delegate to SeaweedOutputStream's position tracking
return outputStream.getPos();
long pos = outputStream.getPos();
LOG.info("[DEBUG-2024] FSDataOutputStream.getPos() override called (append)! Returning: {} for path: {}", pos, path);
return pos;
}
};
} catch (Exception ex) {

Loading…
Cancel
Save