more flexible replication configuration

other/java/hdfs2/README.md

@@ -130,6 +130,15 @@ The test suite covers:
        <name>fs.seaweed.filer.port.grpc</name>
        <value>18888</value>
      </property>
+     <!-- Optional: Replication configuration with three priority levels:
+          1) If set to non-empty value (e.g. "001") - uses that value
+          2) If set to empty string "" - uses SeaweedFS filer's default replication
+          3) If not configured (property not present) - uses HDFS replication parameter
+     -->
+     <!-- <property>
+       <name>fs.seaweed.replication</name>
+       <value>001</value>
+     </property> -->
    </configuration>
    ```
 

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -59,7 +59,7 @@ public class SeaweedFileSystem extends FileSystem {
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
         conf.setInt(FS_SEAWEED_FILER_PORT, port);
 
-        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port + 10000);
 
         setConf(conf);
         this.uri = uri;
@@ -101,9 +101,16 @@ public class SeaweedFileSystem extends FileSystem {
         path = qualify(path);
 
         try {
-            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
+            // Priority: 1) non-empty FS_SEAWEED_REPLICATION, 2) empty string -> filer
+            // default, 3) null -> HDFS replication
+            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION);
+            if (replicaPlacement == null) {
+                // Not configured, use HDFS replication parameter
+                replicaPlacement = String.format("%03d", replication - 1);
+            }
             int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission,
+                    seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@@ -283,7 +290,6 @@ public class SeaweedFileSystem extends FileSystem {
         seaweedFileSystemStore.setOwner(path, owner, group);
     }
 
-
     /**
      * Set permission of a path.
      *
@@ -351,8 +357,7 @@ public class SeaweedFileSystem extends FileSystem {
 
     @Override
     public void createSymlink(final Path target, final Path link,
-                              final boolean createParent) throws
-            IOException {
+            final boolean createParent) throws IOException {
         // Supporting filesystems should override this method
         throw new UnsupportedOperationException(
                 "Filesystem does not support symlinks!");
@@ -485,7 +490,8 @@ public class SeaweedFileSystem extends FileSystem {
      *
      * @param path    Path to modify
      * @param aclSpec List describing modifications, which must include entries
-     *                for user, group, and others for compatibility with permission bits.
+     *                for user, group, and others for compatibility with permission
+     *                bits.
      * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
      *                                       (default outcome).

other/java/hdfs3/README.md

@@ -130,6 +130,15 @@ The test suite covers:
        <name>fs.seaweed.filer.port.grpc</name>
        <value>18888</value>
      </property>
+     <!-- Optional: Replication configuration with three priority levels:
+          1) If set to non-empty value (e.g. "001") - uses that value
+          2) If set to empty string "" - uses SeaweedFS filer's default replication
+          3) If not configured (property not present) - uses HDFS replication parameter
+     -->
+     <!-- <property>
+       <name>fs.seaweed.replication</name>
+       <value>001</value>
+     </property> -->
    </configuration>
    ```
 

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -59,7 +59,7 @@ public class SeaweedFileSystem extends FileSystem {
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
         conf.setInt(FS_SEAWEED_FILER_PORT, port);
 
-        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port + 10000);
 
         setConf(conf);
         this.uri = uri;
@@ -101,9 +101,16 @@ public class SeaweedFileSystem extends FileSystem {
         path = qualify(path);
 
         try {
-            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
+            // Priority: 1) non-empty FS_SEAWEED_REPLICATION, 2) empty string -> filer
+            // default, 3) null -> HDFS replication
+            String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION);
+            if (replicaPlacement == null) {
+                // Not configured, use HDFS replication parameter
+                replicaPlacement = String.format("%03d", replication - 1);
+            }
             int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission,
+                    seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@@ -283,7 +290,6 @@ public class SeaweedFileSystem extends FileSystem {
         seaweedFileSystemStore.setOwner(path, owner, group);
     }
 
-
     /**
      * Set permission of a path.
      *
@@ -351,8 +357,7 @@ public class SeaweedFileSystem extends FileSystem {
 
     @Override
     public void createSymlink(final Path target, final Path link,
-                              final boolean createParent) throws
-            IOException {
+            final boolean createParent) throws IOException {
         // Supporting filesystems should override this method
         throw new UnsupportedOperationException(
                 "Filesystem does not support symlinks!");
@@ -485,7 +490,8 @@ public class SeaweedFileSystem extends FileSystem {
      *
      * @param path    Path to modify
      * @param aclSpec List describing modifications, which must include entries
-     *                for user, group, and others for compatibility with permission bits.
+     *                for user, group, and others for compatibility with permission
+     *                bits.
      * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
      *                                       (default outcome).