From 95bfec4931260eb7b73c97861c4929623c8321d6 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 26 Sep 2020 01:38:48 +0800 Subject: [PATCH 1/3] hadoop: filesystem cannot create file issues: https://github.com/chrislusf/seaweedfs/issues/1494 --- .../src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 53185367a..2d9f3ccb9 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -202,6 +202,8 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); + + filerClient.createEntry(getParentDirectory(path), entry.build()); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); From ac162fc85769cb1b2a1f8694f9644eae7d0ce6c8 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 10 Oct 2020 13:07:27 +0800 Subject: [PATCH 2/3] hdfs: Hadoop on SeaweedFS: create empty file --- .../java/seaweed/hdfs/SeaweedFileSystemStore.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 2d9f3ccb9..c76160bd2 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -8,14 +8,10 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedRead; +import seaweedfs.client.*; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; @@ -202,8 +198,7 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); - - filerClient.createEntry(getParentDirectory(path), entry.build()); + SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); @@ -280,4 +275,4 @@ public class SeaweedFileSystemStore { } -} +} \ No newline at end of file From 4737df597db3664d9d47e7f47ec6204a832e4c78 Mon Sep 17 00:00:00 2001 From: limd Date: Tue, 17 Nov 2020 19:05:57 +0800 Subject: [PATCH 3/3] HCFS: 1. add replication parameter 2. fix close sequence --- .../hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 5 +++-- .../hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 69df55cf1..fb1f4c53b 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; + public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem { @Override public void close() throws IOException { - this.seaweedFileSystemStore.close(); super.close(); + this.seaweedFileSystemStore.close(); } @Override @@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - String replicaPlacement = String.format("%03d", replication - 1); + String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1)); int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE); OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement); return new FSDataOutputStream(outputStream, statistics); diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 69df55cf1..fb1f4c53b 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; + public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem { @Override public void close() throws IOException { - this.seaweedFileSystemStore.close(); super.close(); + this.seaweedFileSystemStore.close(); } @Override @@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - String replicaPlacement = String.format("%03d", replication - 1); + String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1)); int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE); OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement); return new FSDataOutputStream(outputStream, statistics);