HDFS: add tls secured grpc

other/java/client/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.5</version>
+    <version>1.0.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -2,7 +2,14 @@ package seaweedfs.client;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
 
+import javax.net.ssl.SSLException;
+import java.io.File;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
@@ -20,6 +27,16 @@ public class FilerGrpcClient {
         this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
     }
 
+    public FilerGrpcClient(String host, int grpcPort,
+                           String caFilePath,
+                           String clientCertFilePath,
+                           String clientPrivateKeyFilePath) throws SSLException {
+
+        this(NettyChannelBuilder.forAddress(host, grpcPort)
+                .negotiationType(NegotiationType.TLS)
+                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
+    }
+
     public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
         channel = channelBuilder.build();
         blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
@@ -42,4 +59,18 @@ public class FilerGrpcClient {
     public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
         return futureStub;
     }
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                              String clientCertChainFilePath,
+                                              String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.build();
+    }
+
 }

other/java/hdfs/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.0.5</seaweedfs.client.version>
+        <seaweedfs.client.version>1.0.7</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -34,6 +34,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
+    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
+    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
     private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -72,9 +75,19 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         setConf(conf);
         this.uri = uri;
 
+        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
+                    conf.get(FS_SEAWEED_GRPC_CA),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
+        } else {
             seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
         }
 
+    }
+
     @Override
     public FSDataInputStream open(Path path, int bufferSize) throws IOException {
 
@@ -271,6 +284,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Concat existing files together.
+     *
      * @param trg   the path to the target destination.
      * @param psrcs the paths to the sources to use for the concatenation.
      * @throws IOException                   IO failure
@@ -278,7 +292,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      *                                       (default).
      */
     @Override
-    public void concat(final Path trg, final Path [] psrcs) throws IOException {
+    public void concat(final Path trg, final Path[] psrcs) throws IOException {
         throw new UnsupportedOperationException("Not implemented by the " +
                 getClass().getSimpleName() + " FileSystem implementation");
     }
@@ -291,9 +305,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * <li>Fails if path is not closed.</li>
      * <li>Fails if new size is greater than current size.</li>
      * </ul>
+     *
      * @param f         The path to the file to be truncated
      * @param newLength The size the file is to be truncated to
-     *
      * @return <code>true</code> if the file has been truncated to the desired
      * <code>newLength</code> and is immediately available to be reused for
      * write operations such as <code>append</code>, or
@@ -327,6 +341,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Create a snapshot.
+     *
      * @param path         The directory where snapshots will be taken.
      * @param snapshotName The name of the snapshot
      * @return the snapshot path.
@@ -342,6 +357,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Rename a snapshot.
+     *
      * @param path            The directory path where the snapshot was taken
      * @param snapshotOldName Old name of the snapshot
      * @param snapshotNewName New name of the snapshot
@@ -358,6 +374,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Delete a snapshot of a directory.
+     *
      * @param path         The directory that the to-be-deleted snapshot belongs to
      * @param snapshotName The name of the snapshot
      * @throws IOException                   IO failure

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -12,6 +12,7 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;
 
+import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -33,6 +34,13 @@ public class SeaweedFileSystemStore {
         filerClient = new FilerClient(filerGrpcClient);
     }
 
+    public SeaweedFileSystemStore(String host, int port,
+                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }