diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 6d9f41d9e..9ce6036a0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -14,11 +14,13 @@ public class FilerClient extends FilerGrpcClient { private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class); public FilerClient(String filerHost, int filerGrpcPort) { - super(filerHost, filerGrpcPort-10000, filerGrpcPort); + this(filerHost, filerGrpcPort-10000, filerGrpcPort, ""); } + public FilerClient(String filerHost, int filerGrpcPort, String cn) { this(filerHost, filerGrpcPort-10000, filerGrpcPort, cn); } + public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { this(filerHost, filerPort, filerGrpcPort, ""); } - public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { - super(filerHost, filerPort, filerGrpcPort); + public FilerClient(String filerHost, int filerPort, int filerGrpcPort, String cn) { + super(filerHost, filerPort, filerGrpcPort, cn); } public static String toFileId(FilerProto.FileId fid) { diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 0a2e6332e..44977d186 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -8,7 +8,6 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLException; import java.util.HashMap; import java.util.Map; import java.util.Random; @@ -17,14 +16,12 @@ import java.util.concurrent.TimeUnit; public class FilerGrpcClient { private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class); - static SslContext sslContext; + private static final SslContext sslContext; + private static final String protocol; static { - try { - sslContext = FilerSslContext.loadSslContext(); - } catch (SSLException e) { - logger.warn("failed to load ssl context", e); - } + sslContext = FilerSecurityContext.getGrpcSslContext(); + protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http"; } public final int VOLUME_SERVER_ACCESS_DIRECT = 0; @@ -42,19 +39,27 @@ public class FilerGrpcClient { private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; private String filerAddress; - public FilerGrpcClient(String host, int port, int grpcPort) { - this(host, port, grpcPort, sslContext); + public FilerGrpcClient(String host, int port, int grpcPort, String cn) { + this(host, port, grpcPort, cn, sslContext); } - public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) { + public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) { this(sslContext == null ? - ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() + ManagedChannelBuilder.forAddress(host, grpcPort) + .usePlaintext() .maxInboundMessageSize(1024 * 1024 * 1024) : - NettyChannelBuilder.forAddress(host, grpcPort) - .maxInboundMessageSize(1024 * 1024 * 1024) - .negotiationType(NegotiationType.TLS) - .sslContext(sslContext)); + cn.isEmpty() ? + NettyChannelBuilder.forAddress(host, grpcPort) + .maxInboundMessageSize(1024 * 1024 * 1024) + .negotiationType(NegotiationType.TLS) + .sslContext(sslContext) : + NettyChannelBuilder.forAddress(host, grpcPort) + .maxInboundMessageSize(1024 * 1024 * 1024) + .negotiationType(NegotiationType.TLS) + .overrideAuthority(cn) //will not check hostname of the filer server + .sslContext(sslContext) + ); filerAddress = SeaweedUtil.joinHostPort(host, port); @@ -130,12 +135,11 @@ public class FilerGrpcClient { public String getChunkUrl(String chunkId, String url, String publicUrl) { switch (this.volumeServerAccess) { case VOLUME_SERVER_ACCESS_PUBLIC_URL: - return String.format("http://%s/%s", publicUrl, chunkId); + return String.format("%s://%s/%s", protocol, publicUrl, chunkId); case VOLUME_SERVER_ACCESS_FILER_PROXY: - return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId); + return String.format("%s://%s/?proxyChunkId=%s", protocol, this.filerAddress, chunkId); default: - return String.format("http://%s/%s", url, chunkId); + return String.format("%s://%s/%s", protocol, url, chunkId); } } - } diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java b/other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java new file mode 100644 index 000000000..7cd80a35d --- /dev/null +++ b/other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java @@ -0,0 +1,164 @@ +package seaweedfs.client; + +import com.google.common.base.Strings; +import com.moandjiezana.toml.Toml; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.*; +import java.io.File; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; + +public abstract class FilerSecurityContext extends SslContext { +//extends Netty SslContext to access its protected static utility methods in +//buildHttpSslContext() + + private static final Logger logger = LoggerFactory.getLogger(FilerSecurityContext.class); + private static boolean grpcSecurityEnabled; + private static boolean httpSecurityEnabled; + private static SslContext grpcSslContext; + private static SSLContext httpSslContext; + + private static String grpcTrustCertCollectionFilePath; + private static String grpcClientCertChainFilePath; + private static String grpcClientPrivateKeyFilePath; + + private static String httpTrustCertCollectionFilePath; + private static String httpClientCertChainFilePath; + private static String httpClientPrivateKeyFilePath; + + + static { + String securityFileName = "security.toml"; + String home = System.getProperty("user.home"); + File f1 = new File("./"+securityFileName); + File f2 = new File(home + "/.seaweedfs/"+securityFileName); + File f3 = new File("/etc/seaweedfs/"+securityFileName); + + File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null; + + if (securityFile==null){ + logger.debug("Security file not found"); + grpcSecurityEnabled = false; + httpSecurityEnabled = false; + } else { + + Toml toml = new Toml().read(securityFile); + logger.debug("reading ssl setup from {}", securityFile); + + grpcTrustCertCollectionFilePath = toml.getString("grpc.ca"); + logger.debug("loading gRPC ca from {}", grpcTrustCertCollectionFilePath); + grpcClientCertChainFilePath = toml.getString("grpc.client.cert"); + logger.debug("loading gRPC client ca from {}", grpcClientCertChainFilePath); + grpcClientPrivateKeyFilePath = toml.getString("grpc.client.key"); + logger.debug("loading gRPC client key from {}", grpcClientPrivateKeyFilePath); + + if (Strings.isNullOrEmpty(grpcClientCertChainFilePath) && Strings.isNullOrEmpty(grpcClientPrivateKeyFilePath)) { + logger.debug("gRPC private key file locations not set"); + grpcSecurityEnabled = false; + } else { + try { + grpcSslContext = buildGrpcSslContext(); + grpcSecurityEnabled = true; + } catch (Exception e) { + logger.warn("Couldn't initialize gRPC security context, filer operations are likely to fail!", e); + grpcSslContext = null; + grpcSecurityEnabled = false; + } + } + + if (toml.getBoolean("https.client.enabled")) { + httpTrustCertCollectionFilePath = toml.getString("https.client.ca"); + logger.debug("loading HTTP ca from {}", httpTrustCertCollectionFilePath); + httpClientCertChainFilePath = toml.getString("https.client.cert"); + logger.debug("loading HTTP client ca from {}", httpClientCertChainFilePath); + httpClientPrivateKeyFilePath = toml.getString("https.client.key"); + logger.debug("loading HTTP client key from {}", httpClientPrivateKeyFilePath); + + if (Strings.isNullOrEmpty(httpClientCertChainFilePath) && Strings.isNullOrEmpty(httpClientPrivateKeyFilePath)) { + logger.debug("HTTP private key file locations not set"); + httpSecurityEnabled = false; + } else { + try { + httpSslContext = buildHttpSslContext(); + httpSecurityEnabled = true; + } catch (Exception e) { + logger.warn("Couldn't initialize HTTP security context, volume operations are likely to fail!", e); + httpSslContext = null; + httpSecurityEnabled = false; + } + } + } else { + httpSecurityEnabled = false; + } + } + // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html + } + + public static boolean isGrpcSecurityEnabled() { + return grpcSecurityEnabled; + } + + public static boolean isHttpSecurityEnabled() { + return httpSecurityEnabled; + } + + public static SslContext getGrpcSslContext() { + return grpcSslContext; + } + + public static SSLContext getHttpSslContext() { + return httpSslContext; + } + + private static SslContext buildGrpcSslContext() throws SSLException { + SslContextBuilder builder = GrpcSslContexts.forClient(); + if (grpcTrustCertCollectionFilePath != null) { + builder.trustManager(new File(grpcTrustCertCollectionFilePath)); + } + if (grpcClientCertChainFilePath != null && grpcClientPrivateKeyFilePath != null) { + builder.keyManager(new File(grpcClientCertChainFilePath), new File(grpcClientPrivateKeyFilePath)); + } + + return builder.build(); + } + + private static SSLContext buildHttpSslContext() throws GeneralSecurityException, IOException { + SSLContextBuilder builder = SSLContexts.custom(); + + if (httpTrustCertCollectionFilePath != null) { + final X509Certificate[] trustCerts = toX509Certificates(new File(httpTrustCertCollectionFilePath)); + final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + ks.load(null, null); + + int i = 0; + for (X509Certificate cert: trustCerts) { + String alias = Integer.toString(++i); + ks.setCertificateEntry(alias, cert); + } + + builder.loadTrustMaterial(ks, null); + } + + if (httpClientCertChainFilePath != null && httpClientPrivateKeyFilePath != null) { + final X509Certificate[] keyCerts = toX509Certificates(new File(httpClientCertChainFilePath)); + final PrivateKey key = toPrivateKey(new File(httpClientPrivateKeyFilePath), null); + char[] emptyPassword = new char[0]; + final KeyStore ks = buildKeyStore(keyCerts, key, emptyPassword, null); + logger.debug("Loaded {} key certificates", ks.size()); + builder.loadKeyMaterial(ks, emptyPassword); + } + + return builder.build(); + } +} diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java b/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java deleted file mode 100644 index 520adfd38..000000000 --- a/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java +++ /dev/null @@ -1,64 +0,0 @@ -package seaweedfs.client; - -import com.google.common.base.Strings; -import com.moandjiezana.toml.Toml; -import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; -import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; -import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLException; -import java.io.File; - -public class FilerSslContext { - - private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class); - - public static SslContext loadSslContext() throws SSLException { - String securityFileName = "security.toml"; - String home = System.getProperty("user.home"); - File f1 = new File("./"+securityFileName); - File f2 = new File(home + "/.seaweedfs/"+securityFileName); - File f3 = new File("/etc/seaweedfs/"+securityFileName); - - File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null; - - if (securityFile==null){ - return null; - } - - Toml toml = new Toml().read(securityFile); - logger.debug("reading ssl setup from {}", securityFile); - - String trustCertCollectionFilePath = toml.getString("grpc.ca"); - logger.debug("loading ca from {}", trustCertCollectionFilePath); - String clientCertChainFilePath = toml.getString("grpc.client.cert"); - logger.debug("loading client ca from {}", clientCertChainFilePath); - String clientPrivateKeyFilePath = toml.getString("grpc.client.key"); - logger.debug("loading client key from {}", clientPrivateKeyFilePath); - - if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){ - return null; - } - - // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html - - return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath); - } - - - private static SslContext buildSslContext(String trustCertCollectionFilePath, - String clientCertChainFilePath, - String clientPrivateKeyFilePath) throws SSLException { - SslContextBuilder builder = GrpcSslContexts.forClient(); - if (trustCertCollectionFilePath != null) { - builder.trustManager(new File(trustCertCollectionFilePath)); - } - if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) { - builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath)); - } - return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build(); - } -} diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index 027e49b96..3628a10d1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -1,27 +1,64 @@ package seaweedfs.client; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; public class SeaweedUtil { - static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + private static final Logger logger = LoggerFactory.getLogger(SeaweedUtil.class); + static PoolingHttpClientConnectionManager cm; static CloseableHttpClient httpClient; static { + //Apache HTTP client has a terrible API that makes you configure everything twice + //NoopHostnameVerifier is required because SeaweedFS doesn't verify hostnames + //and the servers are likely to have TLS certificates that do not match their hosts + if (FilerSecurityContext.isHttpSecurityEnabled()) { + SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory( + FilerSecurityContext.getHttpSslContext(), + NoopHostnameVerifier.INSTANCE); + + Registry socketFactoryRegistry = + RegistryBuilder.create() + .register("https", sslSocketFactory) + .register("http", new PlainConnectionSocketFactory()) + .build(); + cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + } else { + cm = new PoolingHttpClientConnectionManager(); + } + // Increase max total connection to 200 cm.setMaxTotal(200); // Increase default max connection per route to 20 cm.setDefaultMaxPerRoute(20); - httpClient = HttpClientBuilder.create() + HttpClientBuilder builder = HttpClientBuilder.create() .setConnectionManager(cm) .setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE) - .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE) - .build(); + .setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE); + + if (FilerSecurityContext.isHttpSecurityEnabled()) { + builder.setSSLContext(FilerSecurityContext.getHttpSslContext()); + builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } + + httpClient = builder.build(); } public static CloseableHttpClient getClosableHttpClient() { diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index b6ea4c3bb..58fcaf975 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; + public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn"; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem { setConf(conf); this.uri = uri; - seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf); + String cn = conf.get(FS_SEAWEED_FILER_CN, ""); + seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf); } @Override diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index a73dbeb74..f65c1961b 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -27,8 +27,8 @@ public class SeaweedFileSystemStore { private FilerClient filerClient; private Configuration conf; - public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) { - filerClient = new FilerClient(host, port, grpcPort); + public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) { + filerClient = new FilerClient(host, port, grpcPort, cn); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { @@ -36,7 +36,6 @@ public class SeaweedFileSystemStore { } else if (volumeServerAccessMode.equals("filerProxy")) { filerClient.setAccessVolumeServerByFilerProxy(); } - } public void close() { diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index b6ea4c3bb..58fcaf975 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; + public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn"; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem { setConf(conf); this.uri = uri; - seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf); + String cn = conf.get(FS_SEAWEED_FILER_CN, ""); + seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf); } @Override diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index a73dbeb74..f65c1961b 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -27,8 +27,8 @@ public class SeaweedFileSystemStore { private FilerClient filerClient; private Configuration conf; - public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) { - filerClient = new FilerClient(host, port, grpcPort); + public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) { + filerClient = new FilerClient(host, port, grpcPort, cn); this.conf = conf; String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct"); if (volumeServerAccessMode.equals("publicUrl")) { @@ -36,7 +36,6 @@ public class SeaweedFileSystemStore { } else if (volumeServerAccessMode.equals("filerProxy")) { filerClient.setAccessVolumeServerByFilerProxy(); } - } public void close() {