Merge pull request #35 from chrislusf/master

sync

README.md

@@ -144,9 +144,9 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 
 ## Quick Start ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
-* Run `weed server -dir=. -s3` to start one master, one volume server, one filer, and one S3 gateway, with data stored in current directory.
+* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
-You can also start each module individually. See `weed -h`, or `weed <command> -h` for help.
+Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally or a different machine. That is it!
 
 ## Example: Using Seaweed Object Store ##
 

k8s/seaweedfs/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 2.08
+version: 2.09

k8s/seaweedfs/templates/filer-statefulset.yaml

@@ -101,7 +101,7 @@ spec:
               -port={{ .Values.filer.port }} \
               {{- if .Values.filer.metricsPort }}
               -metricsPort {{ .Values.filer.metricsPort }} \
-              {{- end }}}
+              {{- end }}
               {{- if .Values.filer.redirectOnRead }}
               -redirectOnRead \
               {{- end }}

k8s/seaweedfs/templates/s3-deployment.yaml

@@ -73,7 +73,7 @@ spec:
               -port={{ .Values.s3.port }} \
               {{- if .Values.s3.metricsPort }}
               -metricsPort {{ .Values.s3.metricsPort }} \
-              {{- end }}}
+              {{- end }}
               {{- if .Values.global.enableSecurity }}
               -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
               -key.file=/usr/local/share/ca-certificates/client/tls.key \

k8s/seaweedfs/templates/s3-service.yaml

@@ -14,10 +14,10 @@ spec:
     protocol: TCP
 {{- if .Values.s3.metricsPort }}
   - name: "swfs-s3-metrics"
-    port: {{ .Values.filer.s3 }}
+    port: {{ .Values.s3.metricsPort }}
     targetPort: {{ .Values.s3.metricsPort }}
     protocol: TCP
-{{- end }}}
+{{- end }}
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3

k8s/seaweedfs/templates/s3-servicemonitor.yaml

@@ -15,4 +15,4 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3
-{{- end }}}
+{{- end }}

k8s/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.global.monitoring.enabled }}
+{{- $files := .Files.Glob "dashboards/*.json" }}
+{{- if $files }}
+apiVersion: v1
+kind: ConfigMapList
+items:
+{{- range $path, $fileContents := $files }}
+{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
+- apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
+    namespace: {{ $.Release.Namespace }}
+    labels:
+     grafana_dashboard: "1"
+  data:
+    {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s/seaweedfs/templates/volume-service.yaml

@@ -22,7 +22,7 @@ spec:
     port: {{ .Values.volume.metricsPort }}
     targetPort: {{ .Values.volume.metricsPort }}
     protocol: TCP
-{{- end }}}
+{{- end }}
 selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume

k8s/seaweedfs/templates/volume-servicemonitor.yaml

@@ -15,4 +15,4 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume
-{{- end }}}
+{{- end }}

k8s/seaweedfs/templates/volume-statefulset.yaml

@@ -78,7 +78,7 @@ spec:
                 -port={{ .Values.volume.port }} \
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort {{ .Values.volume.metricsPort }} \
-                {{- end }}}
+                {{- end }}
                 -dir={{ .Values.volume.dir }} \
                 -max={{ .Values.volume.maxVolumes }} \
                 {{- if .Values.volume.rack }}

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "2.08"
+  imageTag: "2.09"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.5.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

weed/server/common.go

@@ -38,10 +38,12 @@ func init() {
 
 func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
 	var bytes []byte
-	if r.FormValue("pretty") != "" {
-		bytes, err = json.MarshalIndent(obj, "", "  ")
-	} else {
-		bytes, err = json.Marshal(obj)
+	if obj != nil {
+		if r.FormValue("pretty") != "" {
+			bytes, err = json.MarshalIndent(obj, "", "  ")
+		} else {
+			bytes, err = json.Marshal(obj)
+		}
 	}
 	if err != nil {
 		return

weed/server/filer_server_handlers.go

@@ -26,11 +26,19 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 		stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds())
 	case "DELETE":
 		stats.FilerRequestCounter.WithLabelValues("delete").Inc()
-		fs.DeleteHandler(w, r)
+		if _, ok := r.URL.Query()["tagging"]; ok {
+			fs.DeleteTaggingHandler(w,r)
+		} else {
+			fs.DeleteHandler(w, r)
+		}
 		stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
 	case "PUT":
 		stats.FilerRequestCounter.WithLabelValues("put").Inc()
-		fs.PostHandler(w, r)
+		if _, ok := r.URL.Query()["tagging"]; ok {
+			fs.PutTaggingHandler(w,r)
+		} else {
+			fs.PostHandler(w, r)
+		}
 		stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
 	case "POST":
 		stats.FilerRequestCounter.WithLabelValues("post").Inc()

weed/server/filer_server_handlers_tagging.go

@@ -0,0 +1,102 @@
+package weed_server
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+// add or replace one file Seaweed- prefixed attributes
+// curl -X PUT -H "Seaweed-Name1: value1" http://localhost:8888/path/to/a/file?tagging
+func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request) {
+
+	ctx := context.Background()
+
+	path := r.URL.Path
+	if strings.HasSuffix(path, "/") {
+		path = path[:len(path)-1]
+	}
+
+	existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
+	if err != nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+	if existingEntry == nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+
+	if existingEntry.Extended == nil {
+		existingEntry.Extended = make(map[string][]byte)
+	}
+
+	for header, values := range r.Header {
+		if strings.HasPrefix(header, needle.PairNamePrefix) {
+			for _, value := range values {
+				existingEntry.Extended[header] = []byte(value)
+			}
+		}
+	}
+
+	if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
+		glog.V(0).Infof("failing to update %s tagging : %v", path, dbErr)
+		writeJsonError(w, r, http.StatusInternalServerError, err)
+		return
+	}
+
+	writeJsonQuiet(w, r, http.StatusAccepted, nil)
+	return
+}
+
+// remove all Seaweed- prefixed attributes
+// curl -X DELETE http://localhost:8888/path/to/a/file?tagging
+func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Request) {
+
+	ctx := context.Background()
+
+	path := r.URL.Path
+	if strings.HasSuffix(path, "/") {
+		path = path[:len(path)-1]
+	}
+
+	existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
+	if err != nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+	if existingEntry == nil {
+		writeJsonError(w, r, http.StatusNotFound, err)
+		return
+	}
+
+	if existingEntry.Extended == nil {
+		existingEntry.Extended = make(map[string][]byte)
+	}
+
+	hasDeletion := false
+	for header, _ := range existingEntry.Extended {
+		if strings.HasPrefix(header, needle.PairNamePrefix) {
+			delete(existingEntry.Extended, header)
+			hasDeletion = true
+		}
+	}
+
+	if !hasDeletion {
+		writeJsonQuiet(w, r, http.StatusNotModified, nil)
+		return
+	}
+
+	if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
+		glog.V(0).Infof("failing to delete %s tagging : %v", path, dbErr)
+		writeJsonError(w, r, http.StatusInternalServerError, err)
+		return
+	}
+
+	writeJsonQuiet(w, r, http.StatusAccepted, nil)
+	return
+}

weed/shell/command_volume_fix_replication.go

@@ -369,18 +369,20 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st
 
 func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
 
-	allSame := true
-	oldest := replicas[0]
-	for _, replica := range replicas {
-		if replica.info.ModifiedAtSecond < oldest.info.ModifiedAtSecond {
-			oldest = replica
-			allSame = false
+	sort.Slice(replicas, func(i, j int) bool {
+		a, b := replicas[i], replicas[j]
+		if a.info.CompactRevision != b.info.CompactRevision {
+			return a.info.CompactRevision < b.info.CompactRevision
 		}
-	}
-	if !allSame {
-		return oldest
-	}
+		if a.info.ModifiedAtSecond != b.info.ModifiedAtSecond {
+			return a.info.ModifiedAtSecond < b.info.ModifiedAtSecond
+		}
+		if a.info.Size != b.info.Size {
+			return a.info.Size < b.info.Size
+		}
+		return false
+	})
+
+	return replicas[0]
 
-	// TODO what if all the replicas have the same timestamp?
-	return oldest
 }

weed/storage/store.go

@@ -472,7 +472,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
 				maxVolumeCount += int(uint64(unclaimedSpaces)/volumeSizeLimit) - 1
 			}
 			diskLocation.MaxVolumeCount = maxVolumeCount
-			glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
+			glog.V(2).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
 				diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
 			hasChanges = hasChanges || currentMaxVolumeCount != diskLocation.MaxVolumeCount
 		}