From 69b2dab9c6a6f0c7f2bb2e2e19f76f7887a5fb83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=B5=B7?= Date: Thu, 25 Mar 2021 18:49:26 +0800 Subject: [PATCH 1/2] add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id --- go.mod | 1 + go.sum | 2 ++ weed/command/scaffold.go | 2 +- weed/sequence/snowflake_sequencer.go | 43 ++++++++++++++++++++++++++++ weed/server/master_server.go | 8 ++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 weed/sequence/snowflake_sequencer.go diff --git a/go.mod b/go.mod index 028d16cf4..e792af37b 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/Shopify/sarama v1.23.1 github.com/aws/aws-sdk-go v1.34.30 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 + github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash v1.1.0 github.com/chrislusf/raft v1.0.4 github.com/coreos/go-semver v0.3.0 // indirect diff --git a/go.sum b/go.sum index b49485dc6..31f5520a8 100644 --- a/go.sum +++ b/go.sum @@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index e990bb3ea..52b547563 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -506,7 +506,7 @@ default = "localhost:8888" # used by maintenance scripts if the scripts needs [master.sequencer] -type = "raft" # Choose [raft|etcd] type for storing the file id sequence +type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence # example : http://127.0.0.1:2379,http://127.0.0.1:2389 sequencer_etcd_urls = "http://127.0.0.1:2379" diff --git a/weed/sequence/snowflake_sequencer.go b/weed/sequence/snowflake_sequencer.go new file mode 100644 index 000000000..d6479fc68 --- /dev/null +++ b/weed/sequence/snowflake_sequencer.go @@ -0,0 +1,43 @@ +package sequence + +import ( + "fmt" + "hash/fnv" + + "github.com/bwmarrin/snowflake" +) + +// a simple snowflake Sequencer +type SnowflakeSequencer struct { + node *snowflake.Node +} + +func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) { + node, err := snowflake.NewNode(int64(hash(nodeid) & 0x3ff)) + if err != nil { + fmt.Println(err) + return nil, err + } + + sequencer := &SnowflakeSequencer{node: node} + return sequencer, nil +} + +func hash(s string) uint32 { + h := fnv.New32a() + h.Write([]byte(s)) + return h.Sum32() +} + +func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 { + return uint64(m.node.Generate().Int64()) +} + +// ignore setmax as we are snowflake +func (m *SnowflakeSequencer) SetMax(seenValue uint64) { +} + +// return a new id as no Peek is stored +func (m *SnowflakeSequencer) Peek() uint64 { + return uint64(m.node.Generate().Int64()) +} diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 9404081b4..f5bad53fa 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -277,6 +277,14 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer glog.Error(err) seq = nil } + case "snowflake": + var err error + glog.V(0).Infof("use a snowfalke seq id, nodeid %s:%d", option.Host, option.Port) + seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port)) + if err != nil { + glog.Error(err) + seq = nil + } default: seq = sequence.NewMemorySequencer() } From 06be5dc6c3cd50c461656e35a53585dd44d94793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=B5=B7?= Date: Thu, 25 Mar 2021 18:59:40 +0800 Subject: [PATCH 2/2] log snowflake sequencer nodeid's hex when start, it'll be in part of new assigned fid --- weed/sequence/snowflake_sequencer.go | 5 ++++- weed/server/master_server.go | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/weed/sequence/snowflake_sequencer.go b/weed/sequence/snowflake_sequencer.go index d6479fc68..679588120 100644 --- a/weed/sequence/snowflake_sequencer.go +++ b/weed/sequence/snowflake_sequencer.go @@ -5,6 +5,7 @@ import ( "hash/fnv" "github.com/bwmarrin/snowflake" + "github.com/chrislusf/seaweedfs/weed/glog" ) // a simple snowflake Sequencer @@ -13,7 +14,9 @@ type SnowflakeSequencer struct { } func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) { - node, err := snowflake.NewNode(int64(hash(nodeid) & 0x3ff)) + nodeid_hash := hash(nodeid) & 0x3ff + glog.V(0).Infof("use snowfalke seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash) + node, err := snowflake.NewNode(int64(nodeid_hash)) if err != nil { fmt.Println(err) return nil, err diff --git a/weed/server/master_server.go b/weed/server/master_server.go index f5bad53fa..e2b2df18d 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -279,7 +279,6 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer } case "snowflake": var err error - glog.V(0).Infof("use a snowfalke seq id, nodeid %s:%d", option.Host, option.Port) seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port)) if err != nil { glog.Error(err)