Browse Source

Merge pull request #35 from chrislusf/master

sync
pull/1620/head
hilimd 4 years ago
committed by GitHub
parent
commit
a38efe2e7b
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      README.md
  2. 2
      k8s/seaweedfs/Chart.yaml
  3. 1856
      k8s/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
  4. 2
      k8s/seaweedfs/templates/filer-statefulset.yaml
  5. 2
      k8s/seaweedfs/templates/s3-deployment.yaml
  6. 4
      k8s/seaweedfs/templates/s3-service.yaml
  7. 2
      k8s/seaweedfs/templates/s3-servicemonitor.yaml
  8. 20
      k8s/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml
  9. 1352
      k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml
  10. 2
      k8s/seaweedfs/templates/volume-service.yaml
  11. 2
      k8s/seaweedfs/templates/volume-servicemonitor.yaml
  12. 2
      k8s/seaweedfs/templates/volume-statefulset.yaml
  13. 2
      k8s/seaweedfs/values.yaml
  14. 2
      other/java/client/pom.xml
  15. 2
      other/java/client/pom.xml.deploy
  16. 2
      other/java/client/pom_debug.xml
  17. 2
      other/java/hdfs2/dependency-reduced-pom.xml
  18. 2
      other/java/hdfs2/pom.xml
  19. 2
      other/java/hdfs3/dependency-reduced-pom.xml
  20. 2
      other/java/hdfs3/pom.xml
  21. 10
      weed/server/common.go
  22. 12
      weed/server/filer_server_handlers.go
  23. 102
      weed/server/filer_server_handlers_tagging.go
  24. 26
      weed/shell/command_volume_fix_replication.go
  25. 2
      weed/storage/store.go

4
README.md

@ -144,9 +144,9 @@ On top of the object store, optional [Filer] can support directories and POSIX a
## Quick Start ##
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=. -s3` to start one master, one volume server, one filer, and one S3 gateway, with data stored in current directory.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
You can also start each module individually. See `weed -h`, or `weed <command> -h` for help.
Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally or a different machine. That is it!
## Example: Using Seaweed Object Store ##

2
k8s/seaweedfs/Chart.yaml

@ -1,4 +1,4 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
version: 2.08
version: 2.09

1856
k8s/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
File diff suppressed because it is too large
View File

2
k8s/seaweedfs/templates/filer-statefulset.yaml

@ -101,7 +101,7 @@ spec:
-port={{ .Values.filer.port }} \
{{- if .Values.filer.metricsPort }}
-metricsPort {{ .Values.filer.metricsPort }} \
{{- end }}}
{{- end }}
{{- if .Values.filer.redirectOnRead }}
-redirectOnRead \
{{- end }}

2
k8s/seaweedfs/templates/s3-deployment.yaml

@ -73,7 +73,7 @@ spec:
-port={{ .Values.s3.port }} \
{{- if .Values.s3.metricsPort }}
-metricsPort {{ .Values.s3.metricsPort }} \
{{- end }}}
{{- end }}
{{- if .Values.global.enableSecurity }}
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-key.file=/usr/local/share/ca-certificates/client/tls.key \

4
k8s/seaweedfs/templates/s3-service.yaml

@ -14,10 +14,10 @@ spec:
protocol: TCP
{{- if .Values.s3.metricsPort }}
- name: "swfs-s3-metrics"
port: {{ .Values.filer.s3 }}
port: {{ .Values.s3.metricsPort }}
targetPort: {{ .Values.s3.metricsPort }}
protocol: TCP
{{- end }}}
{{- end }}
selector:
app: {{ template "seaweedfs.name" . }}
component: s3

2
k8s/seaweedfs/templates/s3-servicemonitor.yaml

@ -15,4 +15,4 @@ spec:
selector:
app: {{ template "seaweedfs.name" . }}
component: s3
{{- end }}}
{{- end }}

20
k8s/seaweedfs/templates/seaweedfs-grafana-dashboard.yaml

@ -0,0 +1,20 @@
{{- if .Values.global.monitoring.enabled }}
{{- $files := .Files.Glob "dashboards/*.json" }}
{{- if $files }}
apiVersion: v1
kind: ConfigMapList
items:
{{- range $path, $fileContents := $files }}
{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
- apiVersion: v1
kind: ConfigMap
metadata:
name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
namespace: {{ $.Release.Namespace }}
labels:
grafana_dashboard: "1"
data:
{{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
{{- end }}
{{- end }}
{{- end }}

1352
k8s/seaweedfs/templates/seaweefs-grafana-dashboard.yaml
File diff suppressed because it is too large
View File

2
k8s/seaweedfs/templates/volume-service.yaml

@ -22,7 +22,7 @@ spec:
port: {{ .Values.volume.metricsPort }}
targetPort: {{ .Values.volume.metricsPort }}
protocol: TCP
{{- end }}}
{{- end }}
selector:
app: {{ template "seaweedfs.name" . }}
component: volume

2
k8s/seaweedfs/templates/volume-servicemonitor.yaml

@ -15,4 +15,4 @@ spec:
selector:
app: {{ template "seaweedfs.name" . }}
component: volume
{{- end }}}
{{- end }}

2
k8s/seaweedfs/templates/volume-statefulset.yaml

@ -78,7 +78,7 @@ spec:
-port={{ .Values.volume.port }} \
{{- if .Values.volume.metricsPort }}
-metricsPort {{ .Values.volume.metricsPort }} \
{{- end }}}
{{- end }}
-dir={{ .Values.volume.dir }} \
-max={{ .Values.volume.maxVolumes }} \
{{- if .Values.volume.rack }}

2
k8s/seaweedfs/values.yaml

@ -4,7 +4,7 @@ global:
registry: ""
repository: ""
imageName: chrislusf/seaweedfs
imageTag: "2.08"
imageTag: "2.09"
imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret
restartPolicy: Always

2
other/java/client/pom.xml

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version>
<version>1.5.3</version>
<parent>
<groupId>org.sonatype.oss</groupId>

2
other/java/client/pom.xml.deploy

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version>
<version>1.5.3</version>
<parent>
<groupId>org.sonatype.oss</groupId>

2
other/java/client/pom_debug.xml

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version>
<version>1.5.3</version>
<parent>
<groupId>org.sonatype.oss</groupId>

2
other/java/hdfs2/dependency-reduced-pom.xml

@ -301,7 +301,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version>
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>
</project>

2
other/java/hdfs2/pom.xml

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version>
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>

2
other/java/hdfs3/dependency-reduced-pom.xml

@ -309,7 +309,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version>
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>
</project>

2
other/java/hdfs3/pom.xml

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version>
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>

10
weed/server/common.go

@ -38,10 +38,12 @@ func init() {
func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
var bytes []byte
if r.FormValue("pretty") != "" {
bytes, err = json.MarshalIndent(obj, "", " ")
} else {
bytes, err = json.Marshal(obj)
if obj != nil {
if r.FormValue("pretty") != "" {
bytes, err = json.MarshalIndent(obj, "", " ")
} else {
bytes, err = json.Marshal(obj)
}
}
if err != nil {
return

12
weed/server/filer_server_handlers.go

@ -26,11 +26,19 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds())
case "DELETE":
stats.FilerRequestCounter.WithLabelValues("delete").Inc()
fs.DeleteHandler(w, r)
if _, ok := r.URL.Query()["tagging"]; ok {
fs.DeleteTaggingHandler(w,r)
} else {
fs.DeleteHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
case "PUT":
stats.FilerRequestCounter.WithLabelValues("put").Inc()
fs.PostHandler(w, r)
if _, ok := r.URL.Query()["tagging"]; ok {
fs.PutTaggingHandler(w,r)
} else {
fs.PostHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
case "POST":
stats.FilerRequestCounter.WithLabelValues("post").Inc()

102
weed/server/filer_server_handlers_tagging.go

@ -0,0 +1,102 @@
package weed_server
import (
"context"
"net/http"
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
)
// add or replace one file Seaweed- prefixed attributes
// curl -X PUT -H "Seaweed-Name1: value1" http://localhost:8888/path/to/a/file?tagging
func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
path := r.URL.Path
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
}
existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
if err != nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry == nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry.Extended == nil {
existingEntry.Extended = make(map[string][]byte)
}
for header, values := range r.Header {
if strings.HasPrefix(header, needle.PairNamePrefix) {
for _, value := range values {
existingEntry.Extended[header] = []byte(value)
}
}
}
if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
glog.V(0).Infof("failing to update %s tagging : %v", path, dbErr)
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
writeJsonQuiet(w, r, http.StatusAccepted, nil)
return
}
// remove all Seaweed- prefixed attributes
// curl -X DELETE http://localhost:8888/path/to/a/file?tagging
func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
path := r.URL.Path
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
}
existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
if err != nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry == nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry.Extended == nil {
existingEntry.Extended = make(map[string][]byte)
}
hasDeletion := false
for header, _ := range existingEntry.Extended {
if strings.HasPrefix(header, needle.PairNamePrefix) {
delete(existingEntry.Extended, header)
hasDeletion = true
}
}
if !hasDeletion {
writeJsonQuiet(w, r, http.StatusNotModified, nil)
return
}
if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
glog.V(0).Infof("failing to delete %s tagging : %v", path, dbErr)
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
writeJsonQuiet(w, r, http.StatusAccepted, nil)
return
}

26
weed/shell/command_volume_fix_replication.go

@ -369,18 +369,20 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st
func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
allSame := true
oldest := replicas[0]
for _, replica := range replicas {
if replica.info.ModifiedAtSecond < oldest.info.ModifiedAtSecond {
oldest = replica
allSame = false
sort.Slice(replicas, func(i, j int) bool {
a, b := replicas[i], replicas[j]
if a.info.CompactRevision != b.info.CompactRevision {
return a.info.CompactRevision < b.info.CompactRevision
}
}
if !allSame {
return oldest
}
if a.info.ModifiedAtSecond != b.info.ModifiedAtSecond {
return a.info.ModifiedAtSecond < b.info.ModifiedAtSecond
}
if a.info.Size != b.info.Size {
return a.info.Size < b.info.Size
}
return false
})
return replicas[0]
// TODO what if all the replicas have the same timestamp?
return oldest
}

2
weed/storage/store.go

@ -472,7 +472,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
maxVolumeCount += int(uint64(unclaimedSpaces)/volumeSizeLimit) - 1
}
diskLocation.MaxVolumeCount = maxVolumeCount
glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
glog.V(2).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
hasChanges = hasChanges || currentMaxVolumeCount != diskLocation.MaxVolumeCount
}

Loading…
Cancel
Save