Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

feature: added ssl support for HCFS (#6699) (#6775)

pull/6777/head
orthoxerox 3 days ago
committed by GitHub
parent
848d1f7c34
commit
d8cc269294
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
9 changed files with 243 additions and 98 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      other/java/client/src/main/java/seaweedfs/client/FilerClient.java
  2. 42
      other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
  3. 164
      other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java
  4. 64
      other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
  5. 45
      other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
  6. 4
      other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
  7. 5
      other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
  8. 4
      other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
  9. 5
      other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

8
other/java/client/src/main/java/seaweedfs/client/FilerClient.java
View File

@ -14,11 +14,13 @@ public class FilerClient extends FilerGrpcClient {
private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
public FilerClient(String filerHost, int filerGrpcPort) {
super(filerHost, filerGrpcPort-10000, filerGrpcPort);
this(filerHost, filerGrpcPort-10000, filerGrpcPort, "");
}
public FilerClient(String filerHost, int filerGrpcPort, String cn) { this(filerHost, filerGrpcPort-10000, filerGrpcPort, cn); }
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) { this(filerHost, filerPort, filerGrpcPort, ""); }
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
super(filerHost, filerPort, filerGrpcPort);
public FilerClient(String filerHost, int filerPort, int filerGrpcPort, String cn) {
super(filerHost, filerPort, filerGrpcPort, cn);
}
public static String toFileId(FilerProto.FileId fid) {

42
other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
View File

@ -8,7 +8,6 @@ import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@ -17,14 +16,12 @@ import java.util.concurrent.TimeUnit;
public class FilerGrpcClient {
private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);
static SslContext sslContext;
private static final SslContext sslContext;
private static final String protocol;
static {
try {
sslContext = FilerSslContext.loadSslContext();
} catch (SSLException e) {
logger.warn("failed to load ssl context", e);
}
sslContext = FilerSecurityContext.getGrpcSslContext();
protocol = FilerSecurityContext.isHttpSecurityEnabled() ? "https" : "http";
}
public final int VOLUME_SERVER_ACCESS_DIRECT = 0;
@ -42,19 +39,27 @@ public class FilerGrpcClient {
private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
private String filerAddress;
public FilerGrpcClient(String host, int port, int grpcPort) {
this(host, port, grpcPort, sslContext);
public FilerGrpcClient(String host, int port, int grpcPort, String cn) {
this(host, port, grpcPort, cn, sslContext);
}
public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
public FilerGrpcClient(String host, int port, int grpcPort, String cn, SslContext sslContext) {
this(sslContext == null ?
ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
ManagedChannelBuilder.forAddress(host, grpcPort)
.usePlaintext()
.maxInboundMessageSize(1024 * 1024 * 1024) :
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.sslContext(sslContext));
cn.isEmpty() ?
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.sslContext(sslContext) :
NettyChannelBuilder.forAddress(host, grpcPort)
.maxInboundMessageSize(1024 * 1024 * 1024)
.negotiationType(NegotiationType.TLS)
.overrideAuthority(cn) //will not check hostname of the filer server
.sslContext(sslContext)
);
filerAddress = SeaweedUtil.joinHostPort(host, port);
@ -130,12 +135,11 @@ public class FilerGrpcClient {
public String getChunkUrl(String chunkId, String url, String publicUrl) {
switch (this.volumeServerAccess) {
case VOLUME_SERVER_ACCESS_PUBLIC_URL:
return String.format("http://%s/%s", publicUrl, chunkId);
return String.format("%s://%s/%s", protocol, publicUrl, chunkId);
case VOLUME_SERVER_ACCESS_FILER_PROXY:
return String.format("http://%s/?proxyChunkId=%s", this.filerAddress, chunkId);
return String.format("%s://%s/?proxyChunkId=%s", protocol, this.filerAddress, chunkId);
default:
return String.format("http://%s/%s", url, chunkId);
return String.format("%s://%s/%s", protocol, url, chunkId);
}
}
}

164
other/java/client/src/main/java/seaweedfs/client/FilerSecurityContext.java
View File

@ -0,0 +1,164 @@
package seaweedfs.client;
import com.google.common.base.Strings;
import com.moandjiezana.toml.Toml;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.SSLContexts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.*;
import java.io.File;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.PrivateKey;
import java.security.cert.X509Certificate;
public abstract class FilerSecurityContext extends SslContext {
//extends Netty SslContext to access its protected static utility methods in
//buildHttpSslContext()
private static final Logger logger = LoggerFactory.getLogger(FilerSecurityContext.class);
private static boolean grpcSecurityEnabled;
private static boolean httpSecurityEnabled;
private static SslContext grpcSslContext;
private static SSLContext httpSslContext;
private static String grpcTrustCertCollectionFilePath;
private static String grpcClientCertChainFilePath;
private static String grpcClientPrivateKeyFilePath;
private static String httpTrustCertCollectionFilePath;
private static String httpClientCertChainFilePath;
private static String httpClientPrivateKeyFilePath;
static {
String securityFileName = "security.toml";
String home = System.getProperty("user.home");
File f1 = new File("./"+securityFileName);
File f2 = new File(home + "/.seaweedfs/"+securityFileName);
File f3 = new File("/etc/seaweedfs/"+securityFileName);
File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
if (securityFile==null){
logger.debug("Security file not found");
grpcSecurityEnabled = false;
httpSecurityEnabled = false;
} else {
Toml toml = new Toml().read(securityFile);
logger.debug("reading ssl setup from {}", securityFile);
grpcTrustCertCollectionFilePath = toml.getString("grpc.ca");
logger.debug("loading gRPC ca from {}", grpcTrustCertCollectionFilePath);
grpcClientCertChainFilePath = toml.getString("grpc.client.cert");
logger.debug("loading gRPC client ca from {}", grpcClientCertChainFilePath);
grpcClientPrivateKeyFilePath = toml.getString("grpc.client.key");
logger.debug("loading gRPC client key from {}", grpcClientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(grpcClientCertChainFilePath) && Strings.isNullOrEmpty(grpcClientPrivateKeyFilePath)) {
logger.debug("gRPC private key file locations not set");
grpcSecurityEnabled = false;
} else {
try {
grpcSslContext = buildGrpcSslContext();
grpcSecurityEnabled = true;
} catch (Exception e) {
logger.warn("Couldn't initialize gRPC security context, filer operations are likely to fail!", e);
grpcSslContext = null;
grpcSecurityEnabled = false;
}
}
if (toml.getBoolean("https.client.enabled")) {
httpTrustCertCollectionFilePath = toml.getString("https.client.ca");
logger.debug("loading HTTP ca from {}", httpTrustCertCollectionFilePath);
httpClientCertChainFilePath = toml.getString("https.client.cert");
logger.debug("loading HTTP client ca from {}", httpClientCertChainFilePath);
httpClientPrivateKeyFilePath = toml.getString("https.client.key");
logger.debug("loading HTTP client key from {}", httpClientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(httpClientCertChainFilePath) && Strings.isNullOrEmpty(httpClientPrivateKeyFilePath)) {
logger.debug("HTTP private key file locations not set");
httpSecurityEnabled = false;
} else {
try {
httpSslContext = buildHttpSslContext();
httpSecurityEnabled = true;
} catch (Exception e) {
logger.warn("Couldn't initialize HTTP security context, volume operations are likely to fail!", e);
httpSslContext = null;
httpSecurityEnabled = false;
}
}
} else {
httpSecurityEnabled = false;
}
}
// possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
}
public static boolean isGrpcSecurityEnabled() {
return grpcSecurityEnabled;
}
public static boolean isHttpSecurityEnabled() {
return httpSecurityEnabled;
}
public static SslContext getGrpcSslContext() {
return grpcSslContext;
}
public static SSLContext getHttpSslContext() {
return httpSslContext;
}
private static SslContext buildGrpcSslContext() throws SSLException {
SslContextBuilder builder = GrpcSslContexts.forClient();
if (grpcTrustCertCollectionFilePath != null) {
builder.trustManager(new File(grpcTrustCertCollectionFilePath));
}
if (grpcClientCertChainFilePath != null && grpcClientPrivateKeyFilePath != null) {
builder.keyManager(new File(grpcClientCertChainFilePath), new File(grpcClientPrivateKeyFilePath));
}
return builder.build();
}
private static SSLContext buildHttpSslContext() throws GeneralSecurityException, IOException {
SSLContextBuilder builder = SSLContexts.custom();
if (httpTrustCertCollectionFilePath != null) {
final X509Certificate[] trustCerts = toX509Certificates(new File(httpTrustCertCollectionFilePath));
final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
int i = 0;
for (X509Certificate cert: trustCerts) {
String alias = Integer.toString(++i);
ks.setCertificateEntry(alias, cert);
}
builder.loadTrustMaterial(ks, null);
}
if (httpClientCertChainFilePath != null && httpClientPrivateKeyFilePath != null) {
final X509Certificate[] keyCerts = toX509Certificates(new File(httpClientCertChainFilePath));
final PrivateKey key = toPrivateKey(new File(httpClientPrivateKeyFilePath), null);
char[] emptyPassword = new char[0];
final KeyStore ks = buildKeyStore(keyCerts, key, emptyPassword, null);
logger.debug("Loaded {} key certificates", ks.size());
builder.loadKeyMaterial(ks, emptyPassword);
}
return builder.build();
}
}

64
other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
View File

@ -1,64 +0,0 @@
package seaweedfs.client;
import com.google.common.base.Strings;
import com.moandjiezana.toml.Toml;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.File;
public class FilerSslContext {
private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
public static SslContext loadSslContext() throws SSLException {
String securityFileName = "security.toml";
String home = System.getProperty("user.home");
File f1 = new File("./"+securityFileName);
File f2 = new File(home + "/.seaweedfs/"+securityFileName);
File f3 = new File("/etc/seaweedfs/"+securityFileName);
File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
if (securityFile==null){
return null;
}
Toml toml = new Toml().read(securityFile);
logger.debug("reading ssl setup from {}", securityFile);
String trustCertCollectionFilePath = toml.getString("grpc.ca");
logger.debug("loading ca from {}", trustCertCollectionFilePath);
String clientCertChainFilePath = toml.getString("grpc.client.cert");
logger.debug("loading client ca from {}", clientCertChainFilePath);
String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
logger.debug("loading client key from {}", clientPrivateKeyFilePath);
if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
return null;
}
// possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
}
private static SslContext buildSslContext(String trustCertCollectionFilePath,
String clientCertChainFilePath,
String clientPrivateKeyFilePath) throws SSLException {
SslContextBuilder builder = GrpcSslContexts.forClient();
if (trustCertCollectionFilePath != null) {
builder.trustManager(new File(trustCertCollectionFilePath));
}
if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
}
return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
}
}

45
other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
View File

@ -1,27 +1,64 @@
package seaweedfs.client;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
public class SeaweedUtil {
static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
private static final Logger logger = LoggerFactory.getLogger(SeaweedUtil.class);
static PoolingHttpClientConnectionManager cm;
static CloseableHttpClient httpClient;
static {
//Apache HTTP client has a terrible API that makes you configure everything twice
//NoopHostnameVerifier is required because SeaweedFS doesn't verify hostnames
//and the servers are likely to have TLS certificates that do not match their hosts
if (FilerSecurityContext.isHttpSecurityEnabled()) {
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(
FilerSecurityContext.getHttpSslContext(),
NoopHostnameVerifier.INSTANCE);
Registry<ConnectionSocketFactory> socketFactoryRegistry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("https", sslSocketFactory)
.register("http", new PlainConnectionSocketFactory())
.build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
} else {
cm = new PoolingHttpClientConnectionManager();
}
// Increase max total connection to 200
cm.setMaxTotal(200);
// Increase default max connection per route to 20
cm.setDefaultMaxPerRoute(20);
httpClient = HttpClientBuilder.create()
HttpClientBuilder builder = HttpClientBuilder.create()
.setConnectionManager(cm)
.setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE)
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE)
.build();
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
if (FilerSecurityContext.isHttpSecurityEnabled()) {
builder.setSSLContext(FilerSecurityContext.getHttpSslContext());
builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
}
httpClient = builder.build();
}
public static CloseableHttpClient getClosableHttpClient() {

4
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf);
this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
String cn = conf.get(FS_SEAWEED_FILER_CN, "");
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
}
@Override

5
other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort);
public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort, cn);
this.conf = conf;
String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
if (volumeServerAccessMode.equals("publicUrl")) {
@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
} else if (volumeServerAccessMode.equals("filerProxy")) {
filerClient.setAccessVolumeServerByFilerProxy();
}
}
public void close() {

4
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
View File

@ -29,6 +29,7 @@ public class SeaweedFileSystem extends FileSystem {
public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
public static final String FS_SEAWEED_VOLUME_SERVER_ACCESS = "fs.seaweed.volume.server.access";
public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
public static final String FS_SEAWEED_FILER_CN = "fs.seaweed.filer.cn";
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
@ -63,8 +64,9 @@ public class SeaweedFileSystem extends FileSystem {
setConf(conf);
this.uri = uri;
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
String cn = conf.get(FS_SEAWEED_FILER_CN, "");
seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, cn, conf);
}
@Override

5
other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
View File

@ -27,8 +27,8 @@ public class SeaweedFileSystemStore {
private FilerClient filerClient;
private Configuration conf;
public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort);
public SeaweedFileSystemStore(String host, int port, int grpcPort, String cn, Configuration conf) {
filerClient = new FilerClient(host, port, grpcPort, cn);
this.conf = conf;
String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
if (volumeServerAccessMode.equals("publicUrl")) {
@ -36,7 +36,6 @@ public class SeaweedFileSystemStore {
} else if (volumeServerAccessMode.equals("filerProxy")) {
filerClient.setAccessVolumeServerByFilerProxy();
}
}
public void close() {

Write Preview
Loading…
Cancel
Save