Browse Source

Merge pull request #1945 from bash99/master

Add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id
pull/1946/head
Chris Lu 4 years ago
committed by GitHub
parent
commit
b8d3f11e92
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      go.mod
  2. 2
      go.sum
  3. 2
      weed/command/scaffold.go
  4. 46
      weed/sequence/snowflake_sequencer.go
  5. 7
      weed/server/master_server.go

1
go.mod

@ -13,6 +13,7 @@ require (
github.com/Shopify/sarama v1.23.1 github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.34.30 github.com/aws/aws-sdk-go v1.34.30
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash v1.1.0
github.com/chrislusf/raft v1.0.4 github.com/chrislusf/raft v1.0.4
github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect

2
go.sum

@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=

2
weed/command/scaffold.go

@ -506,7 +506,7 @@ default = "localhost:8888" # used by maintenance scripts if the scripts needs
[master.sequencer] [master.sequencer]
type = "raft" # Choose [raft|etcd] type for storing the file id sequence
type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence
# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
# example : http://127.0.0.1:2379,http://127.0.0.1:2389 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
sequencer_etcd_urls = "http://127.0.0.1:2379" sequencer_etcd_urls = "http://127.0.0.1:2379"

46
weed/sequence/snowflake_sequencer.go

@ -0,0 +1,46 @@
package sequence
import (
"fmt"
"hash/fnv"
"github.com/bwmarrin/snowflake"
"github.com/chrislusf/seaweedfs/weed/glog"
)
// a simple snowflake Sequencer
type SnowflakeSequencer struct {
node *snowflake.Node
}
func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
nodeid_hash := hash(nodeid) & 0x3ff
glog.V(0).Infof("use snowfalke seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
node, err := snowflake.NewNode(int64(nodeid_hash))
if err != nil {
fmt.Println(err)
return nil, err
}
sequencer := &SnowflakeSequencer{node: node}
return sequencer, nil
}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
return uint64(m.node.Generate().Int64())
}
// ignore setmax as we are snowflake
func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
}
// return a new id as no Peek is stored
func (m *SnowflakeSequencer) Peek() uint64 {
return uint64(m.node.Generate().Int64())
}

7
weed/server/master_server.go

@ -277,6 +277,13 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
glog.Error(err) glog.Error(err)
seq = nil seq = nil
} }
case "snowflake":
var err error
seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
if err != nil {
glog.Error(err)
seq = nil
}
default: default:
seq = sequence.NewMemorySequencer() seq = sequence.NewMemorySequencer()
} }

Loading…
Cancel
Save