non blocking audit log

weed/command/s3.go

@@ -198,6 +198,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	if len(*s3opt.auditLogConfig) > 0 {
 		s3err.InitAuditLog(*s3opt.auditLogConfig)
 	}
+	defer s3err.Logger.Close()
 
 	if *s3opt.tlsPrivateKey != "" {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)

weed/s3api/s3api_object_handlers.go

@@ -261,7 +261,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 			}
 			if auditLog != nil {
 				auditLog.Key = entryName
-				s3err.PostAccessLog(auditLog)
+				go s3err.PostAccessLog(*auditLog)
 			}
 		}
 

weed/s3api/s3err/audit_fluent.go

@@ -48,23 +48,29 @@ type AccessLogHTTP struct {
 const tag = "s3.access"
 
 var (
-	Logger   *fluent.Fluent
-	hostname = os.Getenv("HOSTNAME")
+	Logger       *fluent.Fluent
+	hostname     = os.Getenv("HOSTNAME")
+	environment  = os.Getenv("ENVIRONMENT")
+	fluentConfig *fluent.Config
 )
 
 func InitAuditLog(config string) {
 	configContent, readErr := os.ReadFile(config)
 	if readErr != nil {
-		glog.Fatalf("fail to read fluent config %s : %v", config, readErr)
+		glog.Errorf("fail to read fluent config %s : %v", config, readErr)
+		return
+	}
+	if err := json.Unmarshal(configContent, fluentConfig); err != nil {
+		glog.Errorf("fail to parse fluent config %s : %v", config, err)
+		return
 	}
-	var fluentConfig fluent.Config
-	if err := json.Unmarshal(configContent, &fluentConfig); err != nil {
-		glog.Fatalf("fail to parse fluent config %s : %v", config, err)
+	if len(fluentConfig.TagPrefix) == 0 && len(environment) > 0 {
+		fluentConfig.TagPrefix = environment
 	}
 	var err error
-	Logger, err = fluent.New(fluentConfig)
+	Logger, err = fluent.New(*fluentConfig)
 	if err != nil {
-		glog.Fatalf("fail to load fluent config: %v", err)
+		glog.Errorf("fail to load fluent config: %v", err)
 	}
 }
 
@@ -131,16 +137,16 @@ func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *Acc
 	if len(remoteIP) == 0 {
 		remoteIP = r.RemoteAddr
 	}
-	hostHeader := r.Header.Get("Host")
+	hostHeader := r.Header.Get("X-Forwarded-Host")
 	if len(hostHeader) == 0 {
-		hostHeader = r.URL.Hostname()
+		hostHeader = r.Host
 	}
 	return &AccessLog{
 		HostHeader: hostHeader,
 		RequestID:  r.Header.Get("X-Request-ID"),
 		RemoteIP:   remoteIP,
 		Requester:  r.Header.Get(xhttp.AmzIdentityId),
-		UserAgent:  r.Header.Get("UserAgent"),
+		UserAgent:  r.Header.Get("user-agent"),
 		HostId:     hostname,
 		Bucket:     bucket,
 		HTTPStatus: HTTPStatusCode,
@@ -155,16 +161,18 @@ func PostLog(r *http.Request, HTTPStatusCode int, errorCode ErrorCode) {
 	if Logger == nil {
 		return
 	}
-	if err := Logger.Post(tag, *GetAccessLog(r, HTTPStatusCode, errorCode)); err != nil {
-		glog.Warning("Error while posting log: ", err)
-	}
+	go func(log *AccessLog) {
+		if err := Logger.Post(tag, *log); err != nil {
+			glog.Warning("Error while posting log: ", err)
+		}
+	}(GetAccessLog(r, HTTPStatusCode, errorCode))
 }
 
-func PostAccessLog(log *AccessLog) {
-	if Logger == nil || log == nil {
+func PostAccessLog(log AccessLog) {
+	if Logger == nil || len(log.Key) == 0 {
 		return
 	}
-	if err := Logger.Post(tag, *log); err != nil {
+	if err := Logger.Post(tag, log); err != nil {
 		glog.Warning("Error while posting log: ", err)
 	}
 }