+
+
+
+
diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go
index 648b97f22..f9ef064bc 100644
--- a/weed/server/filer_ui/templates.go
+++ b/weed/server/filer_ui/templates.go
@@ -1,6 +1,7 @@
package filer_ui
import (
+ _ "embed"
"github.com/dustin/go-humanize"
"html/template"
"net/url"
@@ -18,178 +19,7 @@ var funcMap = template.FuncMap{
"printpath": printpath,
}
-var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`
-
-
-
+
+
+
+
+ 
+ SeaweedFS Filer
+
+
+
+
+
+ {{ range $entry := .Breadcrumbs }}
+
+ {{ $entry.Name }}
+
+ {{ end }}
+
+
+
+
+
+
+ {{if .ShouldDisplayLoadMore}}
+
+
+ 
+
+
+
+
-
-
-
-
-`))
+var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(filerHtml))
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index 3e6d9bb9e..50c9dbfdf 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -205,8 +205,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
_, err := stream.Recv()
if err != nil {
glog.V(2).Infof("- client %v: %v", clientName, err)
- stopChan <- true
- break
+ close(stopChan)
+ return
}
}
}()
diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go
index 3a4951cc5..4132ce690 100644
--- a/weed/server/master_grpc_server_volume.go
+++ b/weed/server/master_grpc_server_volume.go
@@ -143,7 +143,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
maxTimeout = time.Second * 10
startTime = time.Now()
)
-
+
for time.Now().Sub(startTime) < maxTimeout {
fid, count, dn, err := ms.Topo.PickForWrite(req.Count, option)
if err == nil {
diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index 838803908..eab41524c 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -26,8 +26,9 @@ import (
)
const (
- SequencerType = "master.sequencer.type"
- SequencerEtcdUrls = "master.sequencer.sequencer_etcd_urls"
+ SequencerType = "master.sequencer.type"
+ SequencerEtcdUrls = "master.sequencer.sequencer_etcd_urls"
+ SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
)
type MasterOption struct {
@@ -97,7 +98,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *Maste
ms := &MasterServer{
option: option,
preallocateSize: preallocateSize,
- vgCh: make(chan *topology.VolumeGrowRequest, 1 << 6),
+ vgCh: make(chan *topology.VolumeGrowRequest, 1<<6),
clientChans: make(map[string]chan *master_pb.VolumeLocation),
grpcDialOption: grpcDialOption,
MasterClient: wdclient.NewMasterClient(grpcDialOption, "master", option.Host, 0, "", peers),
@@ -227,6 +228,7 @@ func (ms *MasterServer) startAdminScripts() {
shellOptions.Masters = &masterAddress
shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(filerHostPort)
+ shellOptions.FilerAddress = filerHostPort
shellOptions.Directory = "/"
if err != nil {
glog.V(0).Infof("failed to parse master.filer.default = %s : %v\n", filerHostPort, err)
@@ -293,7 +295,8 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
}
case "snowflake":
var err error
- seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
+ snowflakeId := v.GetInt(SequencerSnowflakeId)
+ seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port), snowflakeId)
if err != nil {
glog.Error(err)
seq = nil
diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go
index 974b3308f..0609732c7 100644
--- a/weed/server/master_server_handlers.go
+++ b/weed/server/master_server_handlers.go
@@ -123,7 +123,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
Count: writableVolumeCount,
ErrCh: errCh,
}
- if err := <- errCh; err != nil {
+ if err := <-errCh; err != nil {
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("cannot grow volume group! %v", err))
return
}
diff --git a/weed/server/master_ui/master.html b/weed/server/master_ui/master.html
new file mode 100644
index 000000000..e0241d66d
--- /dev/null
+++ b/weed/server/master_ui/master.html
@@ -0,0 +1,110 @@
+
+
+
+
-
-
-
-
-
- SeaweedFS Filer
-
-
-
-
-
- {{ range $entry := .Breadcrumbs }}
-
- {{ $entry.Name }}
-
- {{ end }}
-
-
-
-
-
-
- {{if .ShouldDisplayLoadMore}}
-
-
-
-
-
-
-
+
+
+
diff --git a/weed/server/master_ui/templates.go b/weed/server/master_ui/templates.go
index 31b6353e9..415022b97 100644
--- a/weed/server/master_ui/templates.go
+++ b/weed/server/master_ui/templates.go
@@ -1,115 +1,11 @@
package master_ui
import (
+ _ "embed"
"html/template"
)
-var StatusTpl = template.Must(template.New("status").Parse(`
-
-
-
+
+
+
+
+
+
+ SeaweedFS {{ .Version }}
+
+
+
+
+
+
+
+
+
+
+ Cluster status
+| Volume Size Limit | +{{ .VolumeSizeLimitMB }}MB | +
|---|---|
| Free | +{{ .Topology.Free }} | +
| Max | +{{ .Topology.Max }} | +
| Leader | +{{ .Leader }} | +
| Other Masters | +
+
|
+
+
+
+
+ System Stats
+| Concurrent Connections | +{{ .Counters.Connections.WeekCounter.Sum }} | +
|---|---|
| {{ $key }} | +{{ $val }} | +
+
+
+
+
+
+Topology
+| Data Center | +Rack | +RemoteAddr | +#Volumes | +Volume Ids | +#ErasureCodingShards | +Max | +
|---|---|---|---|---|---|---|
{{ $dc.Id }} |
+ {{ $rack.Id }} | +{{ $dn.Url }} + {{ if ne $dn.PublicUrl $dn.Url }} + / {{ $dn.PublicUrl }} + {{ end }} + | +{{ $dn.Volumes }} | +{{ $dn.VolumeIds}} | +{{ $dn.EcShards }} | +{{ $dn.Max }} | +
-
-
-
-`))
+var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
diff --git a/weed/server/volume_grpc_admin.go b/weed/server/volume_grpc_admin.go
index 2bc108a23..898c3da12 100644
--- a/weed/server/volume_grpc_admin.go
+++ b/weed/server/volume_grpc_admin.go
@@ -225,9 +225,9 @@ func (vs *VolumeServer) VolumeNeedleStatus(ctx context.Context, req *volume_serv
if !hasEcVolume {
return nil, fmt.Errorf("volume not found %d", req.VolumeId)
}
- count, err = vs.store.ReadEcShardNeedle(volumeId, n)
+ count, err = vs.store.ReadEcShardNeedle(volumeId, n, nil)
} else {
- count, err = vs.store.ReadVolumeNeedle(volumeId, n, nil)
+ count, err = vs.store.ReadVolumeNeedle(volumeId, n, nil, nil)
}
if err != nil {
return nil, err
diff --git a/weed/server/volume_grpc_batch_delete.go b/weed/server/volume_grpc_batch_delete.go
index 8e84dc2a8..3645ad9c9 100644
--- a/weed/server/volume_grpc_batch_delete.go
+++ b/weed/server/volume_grpc_batch_delete.go
@@ -8,7 +8,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
- "github.com/chrislusf/seaweedfs/weed/storage/types"
)
func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) {
@@ -30,7 +29,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
n := new(needle.Needle)
volumeId, _ := needle.NewVolumeId(vid)
if req.SkipCookieCheck {
- n.Id, err = types.ParseNeedleId(id_cookie)
+ n.Id, _, err = needle.ParseNeedleIdCookie(id_cookie)
if err != nil {
resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
FileId: fid,
@@ -41,7 +40,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
} else {
n.ParsePath(id_cookie)
cookie := n.Cookie
- if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil); err != nil {
+ if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil, nil); err != nil {
resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
FileId: fid,
Status: http.StatusNotFound,
diff --git a/weed/server/volume_grpc_query.go b/weed/server/volume_grpc_query.go
index 2f4fab96a..349d10097 100644
--- a/weed/server/volume_grpc_query.go
+++ b/weed/server/volume_grpc_query.go
@@ -24,7 +24,7 @@ func (vs *VolumeServer) Query(req *volume_server_pb.QueryRequest, stream volume_
n.ParsePath(id_cookie)
cookie := n.Cookie
- if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil); err != nil {
+ if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil, nil); err != nil {
glog.V(0).Infof("volume query failed to read fid %s: %v", fid, err)
return err
}
diff --git a/weed/server/volume_grpc_remote.go b/weed/server/volume_grpc_remote.go
new file mode 100644
index 000000000..fd9db2246
--- /dev/null
+++ b/weed/server/volume_grpc_remote.go
@@ -0,0 +1,56 @@
+package weed_server
+
+import (
+ "context"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+ "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/chrislusf/seaweedfs/weed/remote_storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_server_pb.FetchAndWriteNeedleRequest) (resp *volume_server_pb.FetchAndWriteNeedleResponse, err error) {
+ resp = &volume_server_pb.FetchAndWriteNeedleResponse{}
+ v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
+ if v == nil {
+ return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+ }
+
+ remoteConf := &filer_pb.RemoteConf{
+ Type: req.RemoteType,
+ Name: req.RemoteName,
+ S3AccessKey: req.S3AccessKey,
+ S3SecretKey: req.S3SecretKey,
+ S3Region: req.S3Region,
+ S3Endpoint: req.S3Endpoint,
+ }
+
+ client, getClientErr := remote_storage.GetRemoteStorage(remoteConf)
+ if getClientErr != nil {
+ return nil, fmt.Errorf("get remote client: %v", getClientErr)
+ }
+
+ remoteStorageLocation := &filer_pb.RemoteStorageLocation{
+ Name: req.RemoteName,
+ Bucket: req.RemoteBucket,
+ Path: req.RemotePath,
+ }
+ data, ReadRemoteErr := client.ReadFile(remoteStorageLocation, req.Offset, req.Size)
+ if ReadRemoteErr != nil {
+ return nil, fmt.Errorf("read from remote %+v: %v", remoteStorageLocation, ReadRemoteErr)
+ }
+
+ n := new(needle.Needle)
+ n.Id = types.NeedleId(req.NeedleId)
+ n.Cookie = types.Cookie(req.Cookie)
+ n.Data, n.DataSize = data, uint32(len(data))
+ // copied from *Needle.prepareWriteBuffer()
+ n.Size = 4 + types.Size(n.DataSize) + 1
+ n.Checksum = needle.NewCRC(n.Data)
+ if _, err = vs.store.WriteVolumeNeedle(v.Id, n, false); err != nil {
+ return nil, fmt.Errorf("write needle %d size %d: %v", req.NeedleId, req.Size, err)
+ }
+
+ return resp, nil
+}
diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go
index f7359ea6b..034521b4b 100644
--- a/weed/server/volume_server.go
+++ b/weed/server/volume_server.go
@@ -17,6 +17,13 @@ import (
)
type VolumeServer struct {
+ inFlightUploadDataSize int64
+ inFlightDownloadDataSize int64
+ concurrentUploadLimit int64
+ concurrentDownloadLimit int64
+ inFlightUploadDataLimitCond *sync.Cond
+ inFlightDownloadDataLimitCond *sync.Cond
+
SeedMasterNodes []string
currentMaster string
pulseSeconds int
@@ -28,17 +35,13 @@ type VolumeServer struct {
needleMapKind storage.NeedleMapKind
FixJpgOrientation bool
- ReadRedirect bool
+ ReadMode string
compactionBytePerSecond int64
metricsAddress string
metricsIntervalSec int
fileSizeLimitBytes int64
isHeartbeating bool
stopChan chan bool
-
- inFlightDataSize int64
- inFlightDataLimitCond *sync.Cond
- concurrentUploadLimit int64
}
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
@@ -50,10 +53,11 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
dataCenter string, rack string,
whiteList []string,
fixJpgOrientation bool,
- readRedirect bool,
+ readMode string,
compactionMBPerSecond int,
fileSizeLimitMB int,
concurrentUploadLimit int64,
+ concurrentDownloadLimit int64,
) *VolumeServer {
v := util.GetViper()
@@ -67,19 +71,21 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
readExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
vs := &VolumeServer{
- pulseSeconds: pulseSeconds,
- dataCenter: dataCenter,
- rack: rack,
- needleMapKind: needleMapKind,
- FixJpgOrientation: fixJpgOrientation,
- ReadRedirect: readRedirect,
- grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.volume"),
- compactionBytePerSecond: int64(compactionMBPerSecond) * 1024 * 1024,
- fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
- isHeartbeating: true,
- stopChan: make(chan bool),
- inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
- concurrentUploadLimit: concurrentUploadLimit,
+ pulseSeconds: pulseSeconds,
+ dataCenter: dataCenter,
+ rack: rack,
+ needleMapKind: needleMapKind,
+ FixJpgOrientation: fixJpgOrientation,
+ ReadMode: readMode,
+ grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.volume"),
+ compactionBytePerSecond: int64(compactionMBPerSecond) * 1024 * 1024,
+ fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
+ isHeartbeating: true,
+ stopChan: make(chan bool),
+ inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)),
+ inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
+ concurrentUploadLimit: concurrentUploadLimit,
+ concurrentDownloadLimit: concurrentDownloadLimit,
}
vs.SeedMasterNodes = masterNodes
@@ -112,6 +118,11 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
return vs
}
+func (vs *VolumeServer) SetStopping() {
+ glog.V(0).Infoln("Stopping volume server...")
+ vs.store.SetStopping()
+}
+
func (vs *VolumeServer) Shutdown() {
glog.V(0).Infoln("Shutting down volume server...")
vs.store.Close()
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index 4527add44..ed7807bb8 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -37,6 +37,11 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
switch r.Method {
case "GET", "HEAD":
stats.ReadRequest()
+ vs.inFlightDownloadDataLimitCond.L.Lock()
+ for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit {
+ glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit)
+ vs.inFlightDownloadDataLimitCond.Wait()
+ }
vs.GetOrHeadHandler(w, r)
case "DELETE":
stats.DeleteRequest()
@@ -45,15 +50,16 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
// wait until in flight data is less than the limit
contentLength := getContentLength(r)
- vs.inFlightDataLimitCond.L.Lock()
- for atomic.LoadInt64(&vs.inFlightDataSize) > vs.concurrentUploadLimit {
- vs.inFlightDataLimitCond.Wait()
+ vs.inFlightUploadDataLimitCond.L.Lock()
+ for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
+ glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
+ vs.inFlightUploadDataLimitCond.Wait()
}
- atomic.AddInt64(&vs.inFlightDataSize, contentLength)
- vs.inFlightDataLimitCond.L.Unlock()
+ atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
+ vs.inFlightUploadDataLimitCond.L.Unlock()
defer func() {
- atomic.AddInt64(&vs.inFlightDataSize, -contentLength)
- vs.inFlightDataLimitCond.Signal()
+ atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
+ vs.inFlightUploadDataLimitCond.Signal()
}()
// processs uploads
diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go
index 3e977cfd4..ae3c0b53f 100644
--- a/weed/server/volume_server_handlers_read.go
+++ b/weed/server/volume_server_handlers_read.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
"io"
"mime"
"net/http"
@@ -12,6 +13,7 @@ import (
"path/filepath"
"strconv"
"strings"
+ "sync/atomic"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
@@ -58,14 +60,53 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
hasVolume := vs.store.HasVolume(volumeId)
_, hasEcVolume := vs.store.FindEcVolume(volumeId)
if !hasVolume && !hasEcVolume {
- if !vs.ReadRedirect {
- glog.V(2).Infoln("volume is not local:", err, r.URL.Path)
+ if vs.ReadMode == "local" {
+ glog.V(0).Infoln("volume is not local:", err, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
lookupResult, err := operation.Lookup(vs.GetMaster, volumeId.String())
glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
- if err == nil && len(lookupResult.Locations) > 0 {
+ if err != nil || len(lookupResult.Locations) <= 0 {
+ glog.V(0).Infoln("lookup error:", err, r.URL.Path)
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ if vs.ReadMode == "proxy" {
+ // proxy client request to target server
+ u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].Url))
+ r.URL.Host = u.Host
+ r.URL.Scheme = u.Scheme
+ request, err := http.NewRequest("GET", r.URL.String(), nil)
+ if err != nil {
+ glog.V(0).Infof("failed to instance http request of url %s: %v", r.URL.String(), err)
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ for k, vv := range r.Header {
+ for _, v := range vv {
+ request.Header.Add(k, v)
+ }
+ }
+
+ response, err := client.Do(request)
+ if err != nil {
+ glog.V(0).Infof("request remote url %s: %v", r.URL.String(), err)
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ defer util.CloseResponse(response)
+ // proxy target response to client
+ for k, vv := range response.Header {
+ for _, v := range vv {
+ w.Header().Add(k, v)
+ }
+ }
+ w.WriteHeader(response.StatusCode)
+ io.Copy(w, response.Body)
+ return
+ } else {
+ // redirect
u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl))
u.Path = fmt.Sprintf("%s/%s,%s", u.Path, vid, fid)
arg := url.Values{}
@@ -74,12 +115,8 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
}
u.RawQuery = arg.Encode()
http.Redirect(w, r, u.String(), http.StatusMovedPermanently)
-
- } else {
- glog.V(2).Infoln("lookup error:", err, r.URL.Path)
- w.WriteHeader(http.StatusNotFound)
+ return
}
- return
}
cookie := n.Cookie
@@ -88,11 +125,22 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
}
var count int
+ var needleSize types.Size
+ onReadSizeFn := func(size types.Size) {
+ needleSize = size
+ atomic.AddInt64(&vs.inFlightDownloadDataSize, int64(needleSize))
+ vs.inFlightDownloadDataLimitCond.L.Unlock()
+ }
if hasVolume {
- count, err = vs.store.ReadVolumeNeedle(volumeId, n, readOption)
+ count, err = vs.store.ReadVolumeNeedle(volumeId, n, readOption, onReadSizeFn)
} else if hasEcVolume {
- count, err = vs.store.ReadEcShardNeedle(volumeId, n)
+ count, err = vs.store.ReadEcShardNeedle(volumeId, n, onReadSizeFn)
}
+ defer func() {
+ atomic.AddInt64(&vs.inFlightDownloadDataSize, -int64(needleSize))
+ vs.inFlightDownloadDataLimitCond.Signal()
+ }()
+
if err != nil && err != storage.ErrorDeleted && r.FormValue("type") != "replicate" && hasVolume {
glog.V(4).Infof("read needle: %v", err)
// start to fix it from other replicas, if not deleted and hasVolume and is not a replicated request
@@ -229,7 +277,7 @@ func conditionallyResizeImages(originalDataReaderSeeker io.ReadSeeker, ext strin
}
func shouldResizeImages(ext string, r *http.Request) (width, height int, mode string, shouldResize bool) {
- if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" {
+ if ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".gif" || ext == ".webp" {
if r.FormValue("width") != "" {
width, _ = strconv.Atoi(r.FormValue("width"))
}
diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go
index 3d752eda6..aeb7d6e65 100644
--- a/weed/server/volume_server_handlers_write.go
+++ b/weed/server/volume_server_handlers_write.go
@@ -1,6 +1,7 @@
package weed_server
import (
+ "bytes"
"errors"
"fmt"
"net/http"
@@ -42,7 +43,10 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
return
}
- reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes)
+ bytesBuffer := bufPool.Get().(*bytes.Buffer)
+ defer bufPool.Put(bytesBuffer)
+
+ reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes, bytesBuffer)
if ne != nil {
writeJsonError(w, r, http.StatusBadRequest, ne)
return
@@ -104,7 +108,7 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
return
}
- _, ok := vs.store.ReadVolumeNeedle(volumeId, n, nil)
+ _, ok := vs.store.ReadVolumeNeedle(volumeId, n, nil, nil)
if ok != nil {
m := make(map[string]uint32)
m["size"] = 0
diff --git a/weed/server/volume_server_ui/templates.go b/weed/server/volume_server_ui/templates.go
index 79f1a14a0..d85eb247a 100644
--- a/weed/server/volume_server_ui/templates.go
+++ b/weed/server/volume_server_ui/templates.go
@@ -1,6 +1,7 @@
package volume_server_ui
import (
+ _ "embed"
"fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"html/template"
@@ -24,193 +25,10 @@ var funcMap = template.FuncMap{
"join": join,
"bytesToHumanReadable": util.BytesToHumanReadable,
"percentFrom": percentFrom,
+ "isNotEmpty": util.IsNotEmpty,
}
-var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`
-
-
-
-
-
-
+//go:embed master.html
+var masterHtml string
-
-
- SeaweedFS {{ .Version }}
-
-
-
-
-
-
-
-
-
-
- Cluster status
-| Volume Size Limit | -{{ .VolumeSizeLimitMB }}MB | -
|---|---|
| Free | -{{ .Topology.Free }} | -
| Max | -{{ .Topology.Max }} | -
| Leader | -{{ .Leader }} | -
| Other Masters | -
|
-
-
-
-
- System Stats
-| Concurrent Connections | -{{ .Counters.Connections.WeekCounter.Sum }} | -
|---|---|
| {{ $key }} | -{{ $val }} | -
-
-
-
-
-
- Topology
-| Data Center | -Rack | -RemoteAddr | -#Volumes | -Volume Ids | -#ErasureCodingShards | -Max | -
|---|---|---|---|---|---|---|
{{ $dc.Id }} |
- {{ $rack.Id }} | -{{ $dn.Url }} - {{ if ne $dn.PublicUrl $dn.Url }} - / {{ $dn.PublicUrl }} - {{ end }} - | -{{ $dn.Volumes }} | -{{ $dn.VolumeIds}} | -{{ $dn.EcShards }} | -{{ $dn.Max }} | -
-
-
-
-`))
+var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(volumeHtml))
diff --git a/weed/server/volume_server_ui/volume.html b/weed/server/volume_server_ui/volume.html
new file mode 100644
index 000000000..91809beb0
--- /dev/null
+++ b/weed/server/volume_server_ui/volume.html
@@ -0,0 +1,197 @@
+
+
+
+
-
-
-
-
-
-
- SeaweedFS {{ .Version }}
-
-
-
-
-
-
-
-
-
-
- Disk Stats
-| Path | -Disk | -Total | -Free | -Usage | -
|---|---|---|---|---|
| {{ .Dir }} | -{{ .DiskType }} | -{{ bytesToHumanReadable .All }} | -{{ bytesToHumanReadable .Free }} | -{{ percentFrom .All .Used}}% | -
-
-
-
- System Stats
-| Masters | -{{.Masters}} | -
|---|---|
| Weekly # ReadRequests | -{{ .Counters.ReadRequests.WeekCounter.ToList | join }} | -
| Daily # ReadRequests | -{{ .Counters.ReadRequests.DayCounter.ToList | join }} | -
| Hourly # ReadRequests | -{{ .Counters.ReadRequests.HourCounter.ToList | join }} | -
| Last Minute # ReadRequests | -{{ .Counters.ReadRequests.MinuteCounter.ToList | join }} | -
| {{ $key }} | -{{ $val }} | -
-
-
-
-
-
- Volumes
-| Id | -Collection | -Disk | -Data Size | -Files | -Trash | -TTL | -ReadOnly | -
|---|---|---|---|---|---|---|---|
{{ .Id }} |
- {{ .Collection }} | -{{ .DiskType }} | -{{ bytesToHumanReadable .Size }} | -{{ .FileCount }} | -{{ .DeleteCount }} / {{bytesToHumanReadable .DeletedByteCount}} | -{{ .Ttl }} | -{{ .ReadOnly }} | -
-
-
-
-
-
- Remote Volumes
-| Id | -Collection | -Size | -Files | -Trash | -Remote | -Key | -
|---|---|---|---|---|---|---|
{{ .Id }} |
- {{ .Collection }} | -{{ bytesToHumanReadable .Size }} | -{{ .FileCount }} | -{{ .DeleteCount }} / {{bytesToHumanReadable .DeletedByteCount}} | -{{ .RemoteStorageName }} | -{{ .RemoteStorageKey }} | -
-
-
-
-
+//go:embed volume.html
+var volumeHtml string
- Erasure Coding Shards
-| Id | -Collection | -Shard Size | -Shards | -CreatedAt | -
|---|---|---|---|---|
{{ .VolumeId }} |
- {{ .Collection }} | -{{ bytesToHumanReadable .ShardSize }} | -{{ .ShardIdList }} | -{{ .CreatedAt.Format "02 Jan 06 15:04 -0700" }} | -
+
+
+
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index c6550a36f..68c1f3233 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -532,7 +532,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
return 0, io.EOF
}
if f.entryViewCache == nil {
- f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks)
+ f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks, 0, math.MaxInt64)
f.reader = nil
}
if f.reader == nil {
diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go
index 2a114e61b..ba502a6b9 100644
--- a/weed/shell/command_collection_list.go
+++ b/weed/shell/command_collection_list.go
@@ -22,6 +22,14 @@ func (c *commandCollectionList) Help() string {
return `list all collections`
}
+type CollectionInfo struct {
+ FileCount uint64
+ DeleteCount uint64
+ DeletedByteCount uint64
+ Size uint64
+ VolumeCount int
+}
+
func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
collections, err := ListCollectionNames(commandEnv, true, true)
@@ -30,8 +38,21 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer
return err
}
+ topologyInfo, _, err := collectTopologyInfo(commandEnv)
+ if err != nil {
+ return err
+ }
+
+ collectionInfos := make(map[string]*CollectionInfo)
+
+ writeCollectionInfo(writer, topologyInfo, collectionInfos)
+
for _, c := range collections {
- fmt.Fprintf(writer, "collection:\"%s\"\n", c)
+ cif, found := collectionInfos[c]
+ if !found {
+ continue
+ }
+ fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%d\tfileCount:%d\tdeletedBytes:%d\tdeletion:%d\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount)
}
fmt.Fprintf(writer, "Total %d collections.\n", len(collections))
@@ -56,3 +77,34 @@ func ListCollectionNames(commandEnv *CommandEnv, includeNormalVolumes, includeEc
}
return
}
+
+func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb.VolumeInformationMessage) {
+ c := vif.Collection
+ cif, found := collectionInfos[c]
+ if !found {
+ cif = &CollectionInfo{}
+ collectionInfos[c] = cif
+ }
+ cif.Size += vif.Size
+ cif.DeleteCount += vif.DeleteCount
+ cif.FileCount += vif.FileCount
+ cif.DeletedByteCount += vif.DeletedByteCount
+ cif.VolumeCount++
+}
+
+func writeCollectionInfo(writer io.Writer, t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) {
+ for _, dc := range t.DataCenterInfos {
+ for _, r := range dc.RackInfos {
+ for _, dn := range r.DataNodeInfos {
+ for _, diskInfo := range dn.DiskInfos {
+ for _, vi := range diskInfo.VolumeInfos {
+ addToCollection(collectionInfos, vi)
+ }
+ //for _, ecShardInfo := range diskInfo.EcShardInfos {
+ //
+ //}
+ }
+ }
+ }
+ }
+}
diff --git a/weed/shell/command_ec_decode.go b/weed/shell/command_ec_decode.go
index dafdb041a..e4d597d84 100644
--- a/weed/shell/command_ec_decode.go
+++ b/weed/shell/command_ec_decode.go
@@ -223,21 +223,6 @@ func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyIn
}
-func collectEcShardInfos(topoInfo *master_pb.TopologyInfo, selectedCollection string, vid needle.VolumeId) (ecShardInfos []*master_pb.VolumeEcShardInformationMessage) {
-
- eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
- if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found {
- for _, v := range diskInfo.EcShardInfos {
- if v.Collection == selectedCollection && v.Id == uint32(vid) {
- ecShardInfos = append(ecShardInfos, v)
- }
- }
- }
- })
-
- return
-}
-
func collectEcShardIds(topoInfo *master_pb.TopologyInfo, selectedCollection string) (vids []needle.VolumeId) {
vidMap := make(map[uint32]bool)
diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go
index 02cd7ac69..0aae51d74 100644
--- a/weed/shell/command_fs_configure.go
+++ b/weed/shell/command_fs_configure.go
@@ -30,17 +30,17 @@ func (c *commandFsConfigure) Help() string {
fs.configure
# trying the changes and see the possible configuration file content
- fs.configure -locationPrfix=/my/folder -collection=abc
- fs.configure -locationPrfix=/my/folder -collection=abc -ttl=7d
+ fs.configure -locationPrefix=/my/folder -collection=abc
+ fs.configure -locationPrefix=/my/folder -collection=abc -ttl=7d
# example: configure adding only 1 physical volume for each bucket collection
- fs.configure -locationPrfix=/buckets/ -volumeGrowthCount=1
+ fs.configure -locationPrefix=/buckets/ -volumeGrowthCount=1
# apply the changes
- fs.configure -locationPrfix=/my/folder -collection=abc -apply
+ fs.configure -locationPrefix=/my/folder -collection=abc -apply
# delete the changes
- fs.configure -locationPrfix=/my/folder -delete -apply
+ fs.configure -locationPrefix=/my/folder -delete -apply
`
}
@@ -54,6 +54,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl")
diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|
+
+
+
+
+
+
+ SeaweedFS {{ .Version }}
+
+
+
+
+
+
+
+
+
+
+ Disk Stats
+| Path | +Disk | +Total | +Free | +Usage | +
|---|---|---|---|---|
| {{ .Dir }} | +{{ .DiskType }} | +{{ bytesToHumanReadable .All }} | +{{ bytesToHumanReadable .Free }} | +{{ percentFrom .All .Used}}% | +
+
+
+
+ System Stats
+| Masters | +{{.Masters}} | +
|---|---|
| Weekly # ReadRequests | +{{ .Counters.ReadRequests.WeekCounter.ToList | join }} + | +
| Daily # ReadRequests | +{{ .Counters.ReadRequests.DayCounter.ToList | join }} + | +
| Hourly # ReadRequests | +{{ .Counters.ReadRequests.HourCounter.ToList | join }} + | +
| Last Minute # ReadRequests | +{{ .Counters.ReadRequests.MinuteCounter.ToList | join }} + | +
| {{ $key }} | +{{ $val }} | +
+
+
+
+
+
+ {{ if isNotEmpty .RemoteVolumes }}
+ Volumes
+| Id | +Collection | +Disk | +Data Size | +Files | +Trash | +TTL | +ReadOnly | +
|---|---|---|---|---|---|---|---|
{{ .Id }} |
+ {{ .Collection }} | +{{ .DiskType }} | +{{ bytesToHumanReadable .Size }} | +{{ .FileCount }} | +{{ .DeleteCount }} / {{bytesToHumanReadable .DeletedByteCount}} | +{{ .Ttl }} | +{{ .ReadOnly }} | +
+
+
+
+
+ {{ end }}
+
+ {{ if isNotEmpty .EcVolumes }}
+ Remote Volumes
+| Id | +Collection | +Size | +Files | +Trash | +Remote | +Key | +
|---|---|---|---|---|---|---|
{{ .Id }} |
+ {{ .Collection }} | +{{ bytesToHumanReadable .Size }} | +{{ .FileCount }} | +{{ .DeleteCount }} / {{bytesToHumanReadable .DeletedByteCount}} | +{{ .RemoteStorageName }} | +{{ .RemoteStorageKey }} | +
+
+
+
+
+ {{ end }}
+Erasure Coding Shards
+| Id | +Collection | +Shard Size | +Shards | +CreatedAt | +
|---|---|---|---|---|
{{ .VolumeId }} |
+ {{ .Collection }} | +{{ bytesToHumanReadable .ShardSize }} | +{{ .ShardIdList }} | +{{ .CreatedAt.Format "02 Jan 06 15:04 -0700" }} | +