Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

more flexible replication configuration

pull/7526/head
chrislu 1 week ago
parent
c89f394aba
commit
c96448f3a5
4 changed files with 82 additions and 52 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 9
      other/java/hdfs2/README.md
  2. 20
      other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
  3. 9
      other/java/hdfs3/README.md
  4. 20
      other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

9
other/java/hdfs2/README.md
View File

@ -130,6 +130,15 @@ The test suite covers:
<name>fs.seaweed.filer.port.grpc</name>
<value>18888</value>
</property>
<!-- Optional: Replication configuration with three priority levels:
1) If set to non-empty value (e.g. "001") - uses that value
2) If set to empty string "" - uses SeaweedFS filer's default replication
3) If not configured (property not present) - uses HDFS replication parameter
-->
<!-- <property>
<name>fs.seaweed.replication</name>
<value>001</value>
</property> -->
</configuration>
```

20
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -59,7 +59,7 @@ public class SeaweedFileSystem extends FileSystem {
port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
conf.setInt(FS_SEAWEED_FILER_PORT, port);
int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port + 10000);
setConf(conf);
this.uri = uri;
@ -101,9 +101,16 @@ public class SeaweedFileSystem extends FileSystem {
path = qualify(path);
try {
String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
// Priority: 1) non-empty FS_SEAWEED_REPLICATION, 2) empty string -> filer
// default, 3) null -> HDFS replication
String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION);
if (replicaPlacement == null) {
// Not configured, use HDFS replication parameter
replicaPlacement = String.format("%03d", replication - 1);
}
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission,
seaweedBufferSize, replicaPlacement);
return new FSDataOutputStream(outputStream, statistics);
} catch (Exception ex) {
LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@ -283,7 +290,6 @@ public class SeaweedFileSystem extends FileSystem {
seaweedFileSystemStore.setOwner(path, owner, group);
}
/**
* Set permission of a path.
*
@ -351,8 +357,7 @@ public class SeaweedFileSystem extends FileSystem {
@Override
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws
IOException {
final boolean createParent) throws IOException {
// Supporting filesystems should override this method
throw new UnsupportedOperationException(
"Filesystem does not support symlinks!");
@ -485,7 +490,8 @@ public class SeaweedFileSystem extends FileSystem {
*
* @param path Path to modify
* @param aclSpec List describing modifications, which must include entries
* for user, group, and others for compatibility with permission bits.
* for user, group, and others for compatibility with permission
* bits.
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).

9
other/java/hdfs3/README.md
View File

@ -130,6 +130,15 @@ The test suite covers:
<name>fs.seaweed.filer.port.grpc</name>
<value>18888</value>
</property>
<!-- Optional: Replication configuration with three priority levels:
1) If set to non-empty value (e.g. "001") - uses that value
2) If set to empty string "" - uses SeaweedFS filer's default replication
3) If not configured (property not present) - uses HDFS replication parameter
-->
<!-- <property>
<name>fs.seaweed.replication</name>
<value>001</value>
</property> -->
</configuration>
```

20
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -59,7 +59,7 @@ public class SeaweedFileSystem extends FileSystem {
port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
conf.setInt(FS_SEAWEED_FILER_PORT, port);
int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port + 10000);
setConf(conf);
this.uri = uri;
@ -101,9 +101,16 @@ public class SeaweedFileSystem extends FileSystem {
path = qualify(path);
try {
String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1));
// Priority: 1) non-empty FS_SEAWEED_REPLICATION, 2) empty string -> filer
// default, 3) null -> HDFS replication
String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION);
if (replicaPlacement == null) {
// Not configured, use HDFS replication parameter
replicaPlacement = String.format("%03d", replication - 1);
}
int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission,
seaweedBufferSize, replicaPlacement);
return new FSDataOutputStream(outputStream, statistics);
} catch (Exception ex) {
LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@ -283,7 +290,6 @@ public class SeaweedFileSystem extends FileSystem {
seaweedFileSystemStore.setOwner(path, owner, group);
}
/**
* Set permission of a path.
*
@ -351,8 +357,7 @@ public class SeaweedFileSystem extends FileSystem {
@Override
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws
IOException {
final boolean createParent) throws IOException {
// Supporting filesystems should override this method
throw new UnsupportedOperationException(
"Filesystem does not support symlinks!");
@ -485,7 +490,8 @@ public class SeaweedFileSystem extends FileSystem {
*
* @param path Path to modify
* @param aclSpec List describing modifications, which must include entries
* for user, group, and others for compatibility with permission bits.
* for user, group, and others for compatibility with permission
* bits.
* @throws IOException if an ACL could not be modified
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).

Write Preview
Loading…
Cancel
Save