From a0931be0c0d0cbb7ea5fe778ff6279fd603c06b8 Mon Sep 17 00:00:00 2001
From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com>
Date: Mon, 5 Jun 2023 02:27:56 +0500
Subject: [PATCH] S3 TLS credentials Refreshing (#4506)

* S3 TLS credentials Refreshing

* fix: logging

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
---
 weed/command/s3.go   | 23 +++++++++++++++++++++--
 weed/security/tls.go | 13 ++++++++-----
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/weed/command/s3.go b/weed/command/s3.go
index 8f82ac946..7a599cc86 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -2,8 +2,11 @@ package command
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
 	"google.golang.org/grpc/reflection"
 	"net/http"
 	"time"
@@ -40,6 +43,7 @@ type S3Options struct {
 	auditLogConfig            *string
 	localFilerSocket          *string
 	dataCenter                *string
+	certProvider              certprovider.Provider
 }
 
 func init() {
@@ -150,6 +154,12 @@ func runS3(cmd *Command, args []string) bool {
 
 }
 
+// GetCertificateWithUpdate Auto refreshing TSL certificate
+func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	certs, err := S3opt.certProvider.KeyMaterial(context.Background())
+	return &certs.Certs[0], err
+}
+
 func (s3opt *S3Options) startS3Server() bool {
 
 	filerAddress := pb.ServerAddress(*s3opt.filer)
@@ -245,15 +255,24 @@ func (s3opt *S3Options) startS3Server() bool {
 	go grpcS.Serve(grpcL)
 
 	if *s3opt.tlsPrivateKey != "" {
+		pemfileOptions := pemfile.Options{
+			CertFile:        *s3opt.tlsCertificate,
+			KeyFile:         *s3opt.tlsPrivateKey,
+			RefreshDuration: security.CredRefreshingInterval,
+		}
+		if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
+			glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
+		}
+		httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 		if s3ApiLocalListener != nil {
 			go func() {
-				if err = httpS.ServeTLS(s3ApiLocalListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
+				if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
 					glog.Fatalf("S3 API Server Fail to serve: %v", err)
 				}
 			}()
 		}
-		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
+		if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	} else {
diff --git a/weed/security/tls.go b/weed/security/tls.go
index d5384fc51..ae6510219 100644
--- a/weed/security/tls.go
+++ b/weed/security/tls.go
@@ -16,7 +16,7 @@ import (
 	"google.golang.org/grpc"
 )
 
-const credRefreshingInterval = time.Duration(5) * time.Hour
+const CredRefreshingInterval = time.Duration(5) * time.Hour
 
 type Authenticator struct {
 	AllowedWildcardDomain string
@@ -31,7 +31,10 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption
 	serverOptions := pemfile.Options{
 		CertFile:        config.GetString(component + ".cert"),
 		KeyFile:         config.GetString(component + ".key"),
-		RefreshDuration: credRefreshingInterval,
+		RefreshDuration: CredRefreshingInterval,
+	}
+	if serverOptions.CertFile == "" || serverOptions.KeyFile == "" {
+		return nil, nil
 	}
 
 	serverIdentityProvider, err := pemfile.NewProvider(serverOptions)
@@ -42,7 +45,7 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption
 
 	serverRootOptions := pemfile.Options{
 		RootFile:        config.GetString("grpc.ca"),
-		RefreshDuration: credRefreshingInterval,
+		RefreshDuration: CredRefreshingInterval,
 	}
 	serverRootProvider, err := pemfile.NewProvider(serverRootOptions)
 	if err != nil {
@@ -99,7 +102,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
 	clientOptions := pemfile.Options{
 		CertFile:        certFileName,
 		KeyFile:         keyFileName,
-		RefreshDuration: credRefreshingInterval,
+		RefreshDuration: CredRefreshingInterval,
 	}
 	clientProvider, err := pemfile.NewProvider(clientOptions)
 	if err != nil {
@@ -108,7 +111,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
 	}
 	clientRootOptions := pemfile.Options{
 		RootFile:        config.GetString("grpc.ca"),
-		RefreshDuration: credRefreshingInterval,
+		RefreshDuration: CredRefreshingInterval,
 	}
 	clientRootProvider, err := pemfile.NewProvider(clientRootOptions)
 	if err != nil {