\n\n';
+ html += '\n';
+ html += ' ';
+ html += ' ' + i.name + '' + i.percent + '%<\/div>';
+ html += '<\/div>\n<\/td>\n<\/tr>\n';
}
html += '<\/table>\n';
progressArea.innerHTML = html;
if (values.length > 0) {
progressArea.attributes.style.value = '';
}
- console.log('Render Progress', values);
}
function reportProgress(file, percent) {
@@ -289,7 +301,7 @@
}
function handleCreateDir() {
- var dirName = prompt('Directory Name:', '');
+ var dirName = prompt('Folder Name:', '');
dirName = dirName.trim();
if (dirName == null || dirName == '') {
return;
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index 1ad8edf91..83abdaaad 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -113,6 +113,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes {
+ dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
+ ms.Topo.DataNodeRegistration(dcName, rackName, dn)
+
// process heartbeat.Volumes
stats.MasterReceivedHeartbeatCounter.WithLabelValues("Volumes").Inc()
newVolumes, deletedVolumes := ms.Topo.SyncDataNodeRegistration(heartbeat.Volumes, dn)
diff --git a/weed/server/master_grpc_server_raft.go b/weed/server/master_grpc_server_raft.go
new file mode 100644
index 000000000..37491b3df
--- /dev/null
+++ b/weed/server/master_grpc_server_raft.go
@@ -0,0 +1,66 @@
+package weed_server
+
+import (
+ "context"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/cluster"
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ "github.com/hashicorp/raft"
+)
+
+func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_pb.RaftListClusterServersRequest) (*master_pb.RaftListClusterServersResponse, error) {
+ resp := &master_pb.RaftListClusterServersResponse{}
+
+ servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
+
+ for _, server := range servers {
+ resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{
+ Id: string(server.ID),
+ Address: string(server.Address),
+ Suffrage: server.Suffrage.String(),
+ })
+ }
+ return resp, nil
+}
+
+func (ms *MasterServer) RaftAddServer(ctx context.Context, req *master_pb.RaftAddServerRequest) (*master_pb.RaftAddServerResponse, error) {
+ resp := &master_pb.RaftAddServerResponse{}
+ if ms.Topo.HashicorpRaft.State() != raft.Leader {
+ return nil, fmt.Errorf("raft add server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
+ }
+
+ var idxFuture raft.IndexFuture
+ if req.Voter {
+ idxFuture = ms.Topo.HashicorpRaft.AddVoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
+ } else {
+ idxFuture = ms.Topo.HashicorpRaft.AddNonvoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
+ }
+
+ if err := idxFuture.Error(); err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+func (ms *MasterServer) RaftRemoveServer(ctx context.Context, req *master_pb.RaftRemoveServerRequest) (*master_pb.RaftRemoveServerResponse, error) {
+ resp := &master_pb.RaftRemoveServerResponse{}
+
+ if ms.Topo.HashicorpRaft.State() != raft.Leader {
+ return nil, fmt.Errorf("raft remove server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
+ }
+
+ if !req.Force {
+ ms.clientChansLock.RLock()
+ _, ok := ms.clientChans[fmt.Sprintf("%s@%s", cluster.MasterType, req.Id)]
+ ms.clientChansLock.RUnlock()
+ if ok {
+ return resp, fmt.Errorf("raft remove server %s failed: client connection to master exists", req.Id)
+ }
+ }
+
+ idxFuture := ms.Topo.HashicorpRaft.RemoveServer(raft.ServerID(req.Id), 0, 0)
+ if err := idxFuture.Error(); err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index b63e3a418..9f29d4ba7 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -1,6 +1,7 @@
package weed_server
import (
+ "context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/stats"
"net/http"
@@ -17,6 +18,7 @@ import (
"github.com/chrislusf/raft"
"github.com/gorilla/mux"
+ hashicorpRaft "github.com/hashicorp/raft"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
@@ -30,8 +32,9 @@ import (
)
const (
- SequencerType = "master.sequencer.type"
- SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
+ SequencerType = "master.sequencer.type"
+ SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
+ RaftServerRemovalTime = 72 * time.Minute
)
type MasterOption struct {
@@ -62,6 +65,9 @@ type MasterServer struct {
boundedLeaderChan chan int
+ onPeerUpdatDoneCn chan string
+ onPeerUpdatDoneCnExist bool
+
// notifying clients
clientChansLock sync.RWMutex
clientChans map[string]chan *master_pb.KeepConnectedResponse
@@ -112,6 +118,9 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
Cluster: cluster.NewCluster(),
}
ms.boundedLeaderChan = make(chan int, 16)
+ ms.onPeerUpdatDoneCn = make(chan string)
+
+ ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
seq := ms.createSequencer(option)
if nil == seq {
@@ -160,19 +169,41 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
}
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
- ms.Topo.RaftServer = raftServer.raftServer
- ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
- glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
- stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
- if ms.Topo.RaftServer.Leader() != "" {
- glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
- }
- })
+ var raftServerName string
+ if raftServer.raftServer != nil {
+ ms.Topo.RaftServer = raftServer.raftServer
+ ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
+ glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
+ stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
+ if ms.Topo.RaftServer.Leader() != "" {
+ glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
+ }
+ })
+ raftServerName = ms.Topo.RaftServer.Name()
+ } else if raftServer.RaftHashicorp != nil {
+ ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
+ leaderCh := raftServer.RaftHashicorp.LeaderCh()
+ prevLeader := ms.Topo.HashicorpRaft.Leader()
+ go func() {
+ for {
+ select {
+ case isLeader := <-leaderCh:
+ leader := ms.Topo.HashicorpRaft.Leader()
+ glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
+ stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
+ prevLeader = leader
+ }
+ }
+ }()
+ raftServerName = ms.Topo.HashicorpRaft.String()
+ }
if ms.Topo.IsLeader() {
- glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
+ glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
} else {
- if ms.Topo.RaftServer.Leader() != "" {
+ if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
+ } else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
+ glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
}
}
}
@@ -181,31 +212,38 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if ms.Topo.IsLeader() {
f(w, r)
- } else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
- ms.boundedLeaderChan <- 1
- defer func() { <-ms.boundedLeaderChan }()
- targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
- if err != nil {
- writeJsonError(w, r, http.StatusInternalServerError,
- fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
- return
- }
- glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
- proxy := httputil.NewSingleHostReverseProxy(targetUrl)
- director := proxy.Director
- proxy.Director = func(req *http.Request) {
- actualHost, err := security.GetActualRemoteHost(req)
- if err == nil {
- req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
- }
- director(req)
- }
- proxy.Transport = util.Transport
- proxy.ServeHTTP(w, r)
- } else {
- // handle requests locally
+ return
+ }
+ var raftServerLeader string
+ if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
+ raftServerLeader = ms.Topo.RaftServer.Leader()
+ } else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
+ raftServerLeader = string(ms.Topo.HashicorpRaft.Leader())
+ }
+ if raftServerLeader == "" {
f(w, r)
+ return
+ }
+ ms.boundedLeaderChan <- 1
+ defer func() { <-ms.boundedLeaderChan }()
+ targetUrl, err := url.Parse("http://" + raftServerLeader)
+ if err != nil {
+ writeJsonError(w, r, http.StatusInternalServerError,
+ fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
+ return
+ }
+ glog.V(4).Infoln("proxying to leader", raftServerLeader)
+ proxy := httputil.NewSingleHostReverseProxy(targetUrl)
+ director := proxy.Director
+ proxy.Director = func(req *http.Request) {
+ actualHost, err := security.GetActualRemoteHost(req)
+ if err == nil {
+ req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
+ }
+ director(req)
}
+ proxy.Transport = util.Transport
+ proxy.ServeHTTP(w, r)
}
}
@@ -301,3 +339,57 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
}
return seq
}
+
+func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
+ if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
+ return
+ }
+ glog.V(4).Infof("OnPeerUpdate: %+v", update)
+
+ peerAddress := pb.ServerAddress(update.Address)
+ peerName := string(peerAddress)
+ isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
+ if update.IsAdd {
+ if isLeader {
+ raftServerFound := false
+ for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
+ if string(server.ID) == peerName {
+ raftServerFound = true
+ }
+ }
+ if !raftServerFound {
+ glog.V(0).Infof("adding new raft server: %s", peerName)
+ ms.Topo.HashicorpRaft.AddVoter(
+ hashicorpRaft.ServerID(peerName),
+ hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
+ }
+ }
+ if ms.onPeerUpdatDoneCnExist {
+ ms.onPeerUpdatDoneCn <- peerName
+ }
+ } else if isLeader {
+ go func(peerName string) {
+ for {
+ select {
+ case <-time.After(RaftServerRemovalTime):
+ err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+ _, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
+ Id: peerName,
+ Force: false,
+ })
+ return err
+ })
+ if err != nil {
+ glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
+ }
+ return
+ case peerDone := <-ms.onPeerUpdatDoneCn:
+ if peerName == peerDone {
+ return
+ }
+ }
+ }
+ }(peerName)
+ ms.onPeerUpdatDoneCnExist = true
+ }
+}
diff --git a/weed/server/master_server_handlers_ui.go b/weed/server/master_server_handlers_ui.go
index 015bfbd00..d8260d8d2 100644
--- a/weed/server/master_server_handlers_ui.go
+++ b/weed/server/master_server_handlers_ui.go
@@ -5,6 +5,8 @@ import (
"time"
"github.com/chrislusf/raft"
+ hashicorpRaft "github.com/hashicorp/raft"
+
ui "github.com/chrislusf/seaweedfs/weed/server/master_ui"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -13,20 +15,40 @@ import (
func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) {
infos := make(map[string]interface{})
infos["Up Time"] = time.Now().Sub(startTime).String()
- args := struct {
- Version string
- Topology interface{}
- RaftServer raft.Server
- Stats map[string]interface{}
- Counters *stats.ServerStats
- VolumeSizeLimitMB uint32
- }{
- util.Version(),
- ms.Topo.ToMap(),
- ms.Topo.RaftServer,
- infos,
- serverStats,
- ms.option.VolumeSizeLimitMB,
+ infos["Max Volume Id"] = ms.Topo.GetMaxVolumeId()
+ if ms.Topo.RaftServer != nil {
+ args := struct {
+ Version string
+ Topology interface{}
+ RaftServer raft.Server
+ Stats map[string]interface{}
+ Counters *stats.ServerStats
+ VolumeSizeLimitMB uint32
+ }{
+ util.Version(),
+ ms.Topo.ToMap(),
+ ms.Topo.RaftServer,
+ infos,
+ serverStats,
+ ms.option.VolumeSizeLimitMB,
+ }
+ ui.StatusTpl.Execute(w, args)
+ } else if ms.Topo.HashicorpRaft != nil {
+ args := struct {
+ Version string
+ Topology interface{}
+ RaftServer *hashicorpRaft.Raft
+ Stats map[string]interface{}
+ Counters *stats.ServerStats
+ VolumeSizeLimitMB uint32
+ }{
+ util.Version(),
+ ms.Topo.ToMap(),
+ ms.Topo.HashicorpRaft,
+ infos,
+ serverStats,
+ ms.option.VolumeSizeLimitMB,
+ }
+ ui.StatusNewRaftTpl.Execute(w, args)
}
- ui.StatusTpl.Execute(w, args)
}
diff --git a/weed/server/master_ui/masterNewRaft.html b/weed/server/master_ui/masterNewRaft.html
new file mode 100644
index 000000000..32afdceac
--- /dev/null
+++ b/weed/server/master_ui/masterNewRaft.html
@@ -0,0 +1,121 @@
+
+
+
+ SeaweedFS {{ .Version }}
+
+
+
+
+
+
+
+
+ Cluster status
+
+
+
+ | Volume Size Limit |
+ {{ .VolumeSizeLimitMB }}MB |
+
+
+ | Free |
+ {{ .Topology.Free }} |
+
+
+ | Max |
+ {{ .Topology.Max }} |
+
+ {{ with .RaftServer }}
+
+ | Leader |
+ {{ .Leader }} |
+
+
+ | Other Masters |
+
+
+ {{ range $k, $p := .GetConfiguration.Configuration.Servers }}
+ - {{ $p.ID }}
+ {{ end }}
+
+ |
+
+ {{ end }}
+
+
+
+
+
+ System Stats
+
+
+ | Concurrent Connections |
+ {{ .Counters.Connections.WeekCounter.Sum }} |
+
+ {{ range $key, $val := .Stats }}
+
+ | {{ $key }} |
+ {{ $val }} |
+
+ {{ end }}
+
+ Raft Stats
+
+
+ | applied_index |
+ {{ .RaftServer.Stats.applied_index }} |
+
+
+ | last_log_term |
+ {{ .RaftServer.Stats.last_log_term }} |
+
+
+
+
+
+
+ Topology
+
+
+
+ | Data Center |
+ Rack |
+ RemoteAddr |
+ #Volumes |
+ Volume Ids |
+ #ErasureCodingShards |
+ Max |
+
+
+
+ {{ range $dc_index, $dc := .Topology.DataCenters }}
+ {{ range $rack_index, $rack := $dc.Racks }}
+ {{ range $dn_index, $dn := $rack.DataNodes }}
+
+ {{ $dc.Id }} |
+ {{ $rack.Id }} |
+ {{ $dn.Url }}
+ {{ if ne $dn.PublicUrl $dn.Url }}
+ / {{ $dn.PublicUrl }}
+ {{ end }}
+ |
+ {{ $dn.Volumes }} |
+ {{ $dn.VolumeIds}} |
+ {{ $dn.EcShards }} |
+ {{ $dn.Max }} |
+
+ {{ end }}
+ {{ end }}
+ {{ end }}
+
+
+
+
+
+
+
diff --git a/weed/server/master_ui/templates.go b/weed/server/master_ui/templates.go
index 415022b97..a6dcc57d7 100644
--- a/weed/server/master_ui/templates.go
+++ b/weed/server/master_ui/templates.go
@@ -8,4 +8,8 @@ import (
//go:embed master.html
var masterHtml string
+//go:embed masterNewRaft.html
+var masterNewRaftHtml string
+
var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
+var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml))
diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go
new file mode 100644
index 000000000..cc6578bf5
--- /dev/null
+++ b/weed/server/raft_hashicorp.go
@@ -0,0 +1,183 @@
+package weed_server
+
+// https://yusufs.medium.com/creating-distributed-kv-database-by-implementing-raft-consensus-using-golang-d0884eef2e28
+// https://github.com/Jille/raft-grpc-example/blob/cd5bcab0218f008e044fbeee4facdd01b06018ad/application.go#L18
+
+import (
+ "fmt"
+ transport "github.com/Jille/raft-grpc-transport"
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/pb"
+ "github.com/hashicorp/raft"
+ boltdb "github.com/hashicorp/raft-boltdb"
+ "google.golang.org/grpc"
+ "math/rand"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+)
+
+const (
+ ldbFile = "logs.dat"
+ sdbFile = "stable.dat"
+ updatePeersTimeout = 15 * time.Minute
+)
+
+func getPeerIdx(self pb.ServerAddress, mapPeers map[string]pb.ServerAddress) int {
+ peers := make([]pb.ServerAddress, 0, len(mapPeers))
+ for _, peer := range mapPeers {
+ peers = append(peers, peer)
+ }
+ sort.Slice(peers, func(i, j int) bool {
+ return strings.Compare(string(peers[i]), string(peers[j])) < 0
+ })
+ for i, peer := range peers {
+ if string(peer) == string(self) {
+ return i
+ }
+ }
+ return -1
+}
+
+func (s *RaftServer) AddPeersConfiguration() (cfg raft.Configuration) {
+ for _, peer := range s.peers {
+ cfg.Servers = append(cfg.Servers, raft.Server{
+ Suffrage: raft.Voter,
+ ID: raft.ServerID(peer),
+ Address: raft.ServerAddress(peer.ToGrpcAddress()),
+ })
+ }
+ return cfg
+}
+
+func (s *RaftServer) UpdatePeers() {
+ for {
+ select {
+ case isLeader := <-s.RaftHashicorp.LeaderCh():
+ if isLeader {
+ peerLeader := string(s.serverAddr)
+ existsPeerName := make(map[string]bool)
+ for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers {
+ if string(server.ID) == peerLeader {
+ continue
+ }
+ existsPeerName[string(server.ID)] = true
+ }
+ for _, peer := range s.peers {
+ peerName := string(peer)
+ if peerName == peerLeader || existsPeerName[peerName] {
+ continue
+ }
+ glog.V(0).Infof("adding new peer: %s", peerName)
+ s.RaftHashicorp.AddVoter(
+ raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0)
+ }
+ for peer, _ := range existsPeerName {
+ if _, found := s.peers[peer]; !found {
+ glog.V(0).Infof("removing old peer: %s", peer)
+ s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0)
+ }
+ }
+ if _, found := s.peers[peerLeader]; !found {
+ glog.V(0).Infof("removing old leader peer: %s", peerLeader)
+ s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0)
+ }
+ }
+ return
+ case <-time.After(updatePeersTimeout):
+ return
+ }
+ }
+}
+
+func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
+ s := &RaftServer{
+ peers: option.Peers,
+ serverAddr: option.ServerAddr,
+ dataDir: option.DataDir,
+ topo: option.Topo,
+ }
+
+ c := raft.DefaultConfig()
+ c.LocalID = raft.ServerID(s.serverAddr) // TODO maybee the IP:port address will change
+ c.HeartbeatTimeout = time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1))
+ c.ElectionTimeout = option.ElectionTimeout
+ if c.LeaderLeaseTimeout > c.HeartbeatTimeout {
+ c.LeaderLeaseTimeout = c.HeartbeatTimeout
+ }
+ if glog.V(4) {
+ c.LogLevel = "Debug"
+ } else if glog.V(2) {
+ c.LogLevel = "Info"
+ } else if glog.V(1) {
+ c.LogLevel = "Warn"
+ } else if glog.V(0) {
+ c.LogLevel = "Error"
+ }
+
+ if option.RaftBootstrap {
+ os.RemoveAll(path.Join(s.dataDir, ldbFile))
+ os.RemoveAll(path.Join(s.dataDir, sdbFile))
+ os.RemoveAll(path.Join(s.dataDir, "snapshot"))
+ }
+ baseDir := s.dataDir
+
+ ldb, err := boltdb.NewBoltStore(filepath.Join(baseDir, ldbFile))
+ if err != nil {
+ return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "logs.dat"), err)
+ }
+
+ sdb, err := boltdb.NewBoltStore(filepath.Join(baseDir, sdbFile))
+ if err != nil {
+ return nil, fmt.Errorf(`boltdb.NewBoltStore(%q): %v`, filepath.Join(baseDir, "stable.dat"), err)
+ }
+
+ fss, err := raft.NewFileSnapshotStore(baseDir, 3, os.Stderr)
+ if err != nil {
+ return nil, fmt.Errorf(`raft.NewFileSnapshotStore(%q, ...): %v`, baseDir, err)
+ }
+
+ s.TransportManager = transport.New(raft.ServerAddress(s.serverAddr), []grpc.DialOption{option.GrpcDialOption})
+
+ stateMachine := StateMachine{topo: option.Topo}
+ s.RaftHashicorp, err = raft.NewRaft(c, &stateMachine, ldb, sdb, fss, s.TransportManager.Transport())
+ if err != nil {
+ return nil, fmt.Errorf("raft.NewRaft: %v", err)
+ }
+ if option.RaftBootstrap || len(s.RaftHashicorp.GetConfiguration().Configuration().Servers) == 0 {
+ cfg := s.AddPeersConfiguration()
+ // Need to get lock, in case all servers do this at the same time.
+ peerIdx := getPeerIdx(s.serverAddr, s.peers)
+ timeSpeep := time.Duration(float64(c.LeaderLeaseTimeout) * (rand.Float64()*0.25 + 1) * float64(peerIdx))
+ glog.V(0).Infof("Bootstrapping idx: %d sleep: %v new cluster: %+v", peerIdx, timeSpeep, cfg)
+ time.Sleep(timeSpeep)
+ f := s.RaftHashicorp.BootstrapCluster(cfg)
+ if err := f.Error(); err != nil {
+ return nil, fmt.Errorf("raft.Raft.BootstrapCluster: %v", err)
+ }
+ } else {
+ go s.UpdatePeers()
+ }
+
+ ticker := time.NewTicker(c.HeartbeatTimeout * 10)
+ if glog.V(4) {
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ cfuture := s.RaftHashicorp.GetConfiguration()
+ if err = cfuture.Error(); err != nil {
+ glog.Fatalf("error getting config: %s", err)
+ }
+ configuration := cfuture.Configuration()
+ glog.V(4).Infof("Showing peers known by %s:\n%+v", s.RaftHashicorp.String(), configuration.Servers)
+ }
+ }
+ }()
+ }
+
+ return s, nil
+}
diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go
index d559cb691..8c372f0cc 100644
--- a/weed/server/raft_server.go
+++ b/weed/server/raft_server.go
@@ -2,6 +2,9 @@ package weed_server
import (
"encoding/json"
+ transport "github.com/Jille/raft-grpc-transport"
+ "io"
+ "io/ioutil"
"math/rand"
"os"
"path"
@@ -12,6 +15,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/raft"
+ hashicorpRaft "github.com/hashicorp/raft"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/topology"
@@ -26,14 +30,17 @@ type RaftServerOption struct {
RaftResumeState bool
HeartbeatInterval time.Duration
ElectionTimeout time.Duration
+ RaftBootstrap bool
}
type RaftServer struct {
- peers map[string]pb.ServerAddress // initial peers to join with
- raftServer raft.Server
- dataDir string
- serverAddr pb.ServerAddress
- topo *topology.Topology
+ peers map[string]pb.ServerAddress // initial peers to join with
+ raftServer raft.Server
+ RaftHashicorp *hashicorpRaft.Raft
+ TransportManager *transport.Manager
+ dataDir string
+ serverAddr pb.ServerAddress
+ topo *topology.Topology
*raft.GrpcServer
}
@@ -42,6 +49,8 @@ type StateMachine struct {
topo *topology.Topology
}
+var _ hashicorpRaft.FSM = &StateMachine{}
+
func (s StateMachine) Save() ([]byte, error) {
state := topology.MaxVolumeIdCommand{
MaxVolumeId: s.topo.GetMaxVolumeId(),
@@ -61,6 +70,36 @@ func (s StateMachine) Recovery(data []byte) error {
return nil
}
+func (s *StateMachine) Apply(l *hashicorpRaft.Log) interface{} {
+ before := s.topo.GetMaxVolumeId()
+ state := topology.MaxVolumeIdCommand{}
+ err := json.Unmarshal(l.Data, &state)
+ if err != nil {
+ return err
+ }
+ s.topo.UpAdjustMaxVolumeId(state.MaxVolumeId)
+
+ glog.V(1).Infoln("max volume id", before, "==>", s.topo.GetMaxVolumeId())
+ return nil
+}
+
+func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
+ return &topology.MaxVolumeIdCommand{
+ MaxVolumeId: s.topo.GetMaxVolumeId(),
+ }, nil
+}
+
+func (s *StateMachine) Restore(r io.ReadCloser) error {
+ b, err := ioutil.ReadAll(r)
+ if err != nil {
+ return err
+ }
+ if err := s.Recovery(b); err != nil {
+ return err
+ }
+ return nil
+}
+
func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
s := &RaftServer{
peers: option.Peers,
@@ -132,12 +171,17 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
}
func (s *RaftServer) Peers() (members []string) {
- peers := s.raftServer.Peers()
-
- for _, p := range peers {
- members = append(members, p.Name)
+ if s.raftServer != nil {
+ peers := s.raftServer.Peers()
+ for _, p := range peers {
+ members = append(members, p.Name)
+ }
+ } else if s.RaftHashicorp != nil {
+ cfg := s.RaftHashicorp.GetConfiguration()
+ for _, p := range cfg.Configuration().Servers {
+ members = append(members, string(p.ID))
+ }
}
-
return
}
diff --git a/weed/server/raft_server_handlers.go b/weed/server/raft_server_handlers.go
index 7e58f1e92..cc3e6e37f 100644
--- a/weed/server/raft_server_handlers.go
+++ b/weed/server/raft_server_handlers.go
@@ -25,3 +25,11 @@ func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
}
writeJsonQuiet(w, r, http.StatusOK, ret)
}
+
+func (s *RaftServer) StatsRaftHandler(w http.ResponseWriter, r *http.Request) {
+ if s.RaftHashicorp == nil {
+ writeJsonQuiet(w, r, http.StatusNotFound, nil)
+ return
+ }
+ writeJsonQuiet(w, r, http.StatusOK, s.RaftHashicorp.Stats())
+}
diff --git a/weed/shell/command_cluster_check.go b/weed/shell/command_cluster_check.go
index f841bd224..c8fe110ba 100644
--- a/weed/shell/command_cluster_check.go
+++ b/weed/shell/command_cluster_check.go
@@ -48,7 +48,10 @@ func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer i
emptyDiskTypeDiskInfo, emptyDiskTypeFound := topologyInfo.DiskInfos[""]
hddDiskTypeDiskInfo, hddDiskTypeFound := topologyInfo.DiskInfos["hdd"]
- if !emptyDiskTypeFound && !hddDiskTypeFound || emptyDiskTypeDiskInfo.VolumeCount == 0 && hddDiskTypeDiskInfo.VolumeCount == 0 {
+ if !emptyDiskTypeFound && !hddDiskTypeFound {
+ return fmt.Errorf("Need to a hdd disk type!")
+ }
+ if emptyDiskTypeFound && emptyDiskTypeDiskInfo.VolumeCount == 0 || hddDiskTypeFound && hddDiskTypeDiskInfo.VolumeCount == 0 {
return fmt.Errorf("Need to a hdd disk type!")
}
diff --git a/weed/shell/command_cluster_raft_add.go b/weed/shell/command_cluster_raft_add.go
new file mode 100644
index 000000000..e5f3c41c9
--- /dev/null
+++ b/weed/shell/command_cluster_raft_add.go
@@ -0,0 +1,59 @@
+package shell
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ "io"
+)
+
+func init() {
+ Commands = append(Commands, &commandRaftServerAdd{})
+}
+
+type commandRaftServerAdd struct {
+}
+
+func (c *commandRaftServerAdd) Name() string {
+ return "cluster.raft.add"
+}
+
+func (c *commandRaftServerAdd) Help() string {
+ return `add a server to the raft cluster
+
+ Example:
+ cluster.raft.add -id -address -voter
+`
+}
+
+func (c *commandRaftServerAdd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+ raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+ serverId := raftServerAddCommand.String("id", "", "server id")
+ serverAddress := raftServerAddCommand.String("address", "", "server grpc address")
+ serverVoter := raftServerAddCommand.Bool("voter", true, "assign it a vote")
+ if err = raftServerAddCommand.Parse(args); err != nil {
+ return nil
+ }
+
+ if *serverId == "" || *serverAddress == "" {
+ return fmt.Errorf("empty server id or address")
+ }
+
+ err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+ _, err := client.RaftAddServer(context.Background(), &master_pb.RaftAddServerRequest{
+ Id: *serverId,
+ Address: *serverAddress,
+ Voter: *serverVoter,
+ })
+ if err != nil {
+ return fmt.Errorf("raft add server: %v", err)
+ }
+ println("added server", *serverId)
+ return nil
+ })
+
+ return err
+
+}
diff --git a/weed/shell/command_cluster_raft_ps.go b/weed/shell/command_cluster_raft_ps.go
new file mode 100644
index 000000000..ea868db06
--- /dev/null
+++ b/weed/shell/command_cluster_raft_ps.go
@@ -0,0 +1,51 @@
+package shell
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ "io"
+)
+
+func init() {
+ Commands = append(Commands, &commandRaftClusterPs{})
+}
+
+type commandRaftClusterPs struct {
+}
+
+func (c *commandRaftClusterPs) Name() string {
+ return "cluster.raft.ps"
+}
+
+func (c *commandRaftClusterPs) Help() string {
+ return `check current raft cluster status
+
+ cluster.raft.ps
+`
+}
+
+func (c *commandRaftClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+ raftClusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+ if err = raftClusterPsCommand.Parse(args); err != nil {
+ return nil
+ }
+
+ err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+ resp, err := client.RaftListClusterServers(context.Background(), &master_pb.RaftListClusterServersRequest{})
+ if err != nil {
+ return fmt.Errorf("raft list cluster: %v", err)
+ }
+ fmt.Fprintf(writer, "the raft cluster has %d servers\n", len(resp.ClusterServers))
+ for _, server := range resp.ClusterServers {
+ fmt.Fprintf(writer, " * %s %s (%s)\n", server.Id, server.Address, server.Suffrage)
+ }
+
+ return nil
+ })
+
+ return err
+
+}
diff --git a/weed/shell/command_cluster_raft_remove.go b/weed/shell/command_cluster_raft_remove.go
new file mode 100644
index 000000000..532a1469c
--- /dev/null
+++ b/weed/shell/command_cluster_raft_remove.go
@@ -0,0 +1,56 @@
+package shell
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ "io"
+)
+
+func init() {
+ Commands = append(Commands, &commandRaftServerRemove{})
+}
+
+type commandRaftServerRemove struct {
+}
+
+func (c *commandRaftServerRemove) Name() string {
+ return "cluster.raft.remove"
+}
+
+func (c *commandRaftServerRemove) Help() string {
+ return `remove a server from the raft cluster
+
+ Example:
+ cluster.raft.remove -id
+`
+}
+
+func (c *commandRaftServerRemove) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+ raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+ serverId := raftServerAddCommand.String("id", "", "server id")
+ if err = raftServerAddCommand.Parse(args); err != nil {
+ return nil
+ }
+
+ if *serverId == "" {
+ return fmt.Errorf("empty server id")
+ }
+
+ err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+ _, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
+ Id: *serverId,
+ Force: true,
+ })
+ if err != nil {
+ return fmt.Errorf("raft remove server: %v", err)
+ }
+ println("removed server", *serverId)
+ return nil
+ })
+
+ return err
+
+}
diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go
index 4f893df7a..a6dc8f574 100644
--- a/weed/shell/command_s3_clean_uploads.go
+++ b/weed/shell/command_s3_clean_uploads.go
@@ -3,6 +3,7 @@ package shell
import (
"flag"
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"math"
@@ -39,6 +40,8 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
return nil
}
+ signingKey := util.GetViper().GetString("jwt.signing.key")
+
var filerBucketsPath string
filerBucketsPath, err = readFilerBucketsPath(commandEnv)
if err != nil {
@@ -55,14 +58,16 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
}
for _, bucket := range buckets {
- c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo)
+ if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo, signingKey); err != nil {
+ fmt.Fprintf(writer, fmt.Sprintf("failed cleanup uploads for backet %s: %v", bucket, err))
+ }
}
return err
}
-func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error {
+func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration, signingKey string) error {
uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
var staleUploads []string
now := time.Now()
@@ -77,12 +82,17 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
return fmt.Errorf("list uploads under %v: %v", uploadsDir, err)
}
+ var encodedJwt security.EncodedJwt
+ if signingKey != "" {
+ encodedJwt = security.GenJwtForFilerServer(security.SigningKey(signingKey), 15*60)
+ }
+
for _, staleUpload := range staleUploads {
deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload)
fmt.Fprintf(writer, "purge %s\n", deleteUrl)
- err = util.Delete(deleteUrl, "")
- if err != nil {
+ err = util.Delete(deleteUrl, string(encodedJwt))
+ if err != nil && err.Error() != "" {
return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err)
}
}
diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot
new file mode 100644
index 000000000..4a4ca865d
Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.eot differ
diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg
new file mode 100644
index 000000000..25691af8f
--- /dev/null
+++ b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.svg
@@ -0,0 +1,229 @@
+
+
+
\ No newline at end of file
diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf
new file mode 100644
index 000000000..67fa00bf8
Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.ttf differ
diff --git a/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff
new file mode 100644
index 000000000..8c54182aa
Binary files /dev/null and b/weed/static/bootstrap/3.3.1/fonts/glyphicons-halflings-regular.woff differ
diff --git a/weed/topology/cluster_commands.go b/weed/topology/cluster_commands.go
index 152691ccb..1bcc6b449 100644
--- a/weed/topology/cluster_commands.go
+++ b/weed/topology/cluster_commands.go
@@ -1,9 +1,12 @@
package topology
import (
+ "encoding/json"
+ "fmt"
"github.com/chrislusf/raft"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
+ hashicorpRaft "github.com/hashicorp/raft"
)
type MaxVolumeIdCommand struct {
@@ -20,6 +23,7 @@ func (c *MaxVolumeIdCommand) CommandName() string {
return "MaxVolumeId"
}
+// deprecatedCommandApply represents the old interface to apply a command to the server.
func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
topo := server.Context().(*Topology)
before := topo.GetMaxVolumeId()
@@ -29,3 +33,19 @@ func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
return nil, nil
}
+
+func (s *MaxVolumeIdCommand) Persist(sink hashicorpRaft.SnapshotSink) error {
+ b, err := json.Marshal(s)
+ if err != nil {
+ return fmt.Errorf("marshal: %v", err)
+ }
+ _, err = sink.Write(b)
+ if err != nil {
+ sink.Cancel()
+ return fmt.Errorf("sink.Write(): %v", err)
+ }
+ return sink.Close()
+}
+
+func (s *MaxVolumeIdCommand) Release() {
+}
diff --git a/weed/topology/topology.go b/weed/topology/topology.go
index 207c89ad7..aacdfa7d2 100644
--- a/weed/topology/topology.go
+++ b/weed/topology/topology.go
@@ -1,6 +1,7 @@
package topology
import (
+ "encoding/json"
"errors"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
@@ -10,6 +11,7 @@ import (
"time"
"github.com/chrislusf/raft"
+ hashicorpRaft "github.com/hashicorp/raft"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
@@ -40,7 +42,8 @@ type Topology struct {
Configuration *Configuration
- RaftServer raft.Server
+ RaftServer raft.Server
+ HashicorpRaft *hashicorpRaft.Raft
}
func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology {
@@ -76,6 +79,10 @@ func (t *Topology) IsLeader() bool {
return true
}
}
+ } else if t.HashicorpRaft != nil {
+ if t.HashicorpRaft.State() == hashicorpRaft.Leader {
+ return true
+ }
}
return false
}
@@ -85,6 +92,8 @@ func (t *Topology) Leader() (pb.ServerAddress, error) {
for count := 0; count < 3; count++ {
if t.RaftServer != nil {
l = pb.ServerAddress(t.RaftServer.Leader())
+ } else if t.HashicorpRaft != nil {
+ l = pb.ServerAddress(t.HashicorpRaft.Leader())
} else {
return "", errors.New("Raft Server not ready yet!")
}
@@ -124,8 +133,18 @@ func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []*
func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
vid := t.GetMaxVolumeId()
next := vid.Next()
- if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next)); err != nil {
- return 0, err
+ if t.RaftServer != nil {
+ if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next)); err != nil {
+ return 0, err
+ }
+ } else if t.HashicorpRaft != nil {
+ b, err := json.Marshal(NewMaxVolumeIdCommand(next))
+ if err != nil {
+ return 0, fmt.Errorf("failed marshal NewMaxVolumeIdCommand: %+v", err)
+ }
+ if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
+ return 0, future.Error()
+ }
}
return next, nil
}
@@ -283,3 +302,14 @@ func (t *Topology) IncrementalSyncDataNodeRegistration(newVolumes, deletedVolume
return
}
+
+func (t *Topology) DataNodeRegistration(dcName, rackName string ,dn *DataNode){
+ if dn.Parent() != nil{
+ return
+ }
+ // registration to topo
+ dc := t.GetOrCreateDataCenter(dcName)
+ rack := dc.GetOrCreateRack(rackName)
+ rack.LinkChildNode(dn)
+ glog.Infof("[%s] reLink To topo ", dn.Id())
+}
\ No newline at end of file
diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go
index 0f1db74df..99acefaf5 100644
--- a/weed/topology/topology_event_handling.go
+++ b/weed/topology/topology_event_handling.go
@@ -1,6 +1,7 @@
package topology
import (
+ "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"google.golang.org/grpc"
"math/rand"
@@ -84,7 +85,8 @@ func (t *Topology) UnRegisterDataNode(dn *DataNode) {
negativeUsages := dn.GetDiskUsages().negative()
dn.UpAdjustDiskUsageDelta(negativeUsages)
-
+ dn.DeltaUpdateVolumes([]storage.VolumeInfo{}, dn.GetVolumes())
+ dn.DeltaUpdateEcShards([]*erasure_coding.EcVolumeInfo{}, dn.GetEcShards())
if dn.Parent() != nil {
dn.Parent().UnlinkChildNode(dn.Id())
}
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 41482490c..14f3a0b91 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,7 +5,7 @@ import (
)
var (
- VERSION_NUMBER = fmt.Sprintf("%.02f", 2.97)
+ VERSION_NUMBER = fmt.Sprintf("%.02f", 2.98)
VERSION = sizeLimit + " " + VERSION_NUMBER
COMMIT = ""
)
|