Browse Source
1. volume server now sends master server its max file key, so that
1. volume server now sends master server its max file key, so that
master server does not need to store the sequence on disk any more 2. fix raft server's failure to init cluster during bootstrappingpull/2/head
Chris Lu
11 years ago
17 changed files with 60 additions and 256 deletions
-
30go/metastore/backing_test.go
-
34go/metastore/file_backing.go
-
36go/metastore/memory_backing.go
-
33go/metastore/metastore.go
-
19go/sequence/memory_sequencer.go
-
86go/sequence/sequence.go
-
4go/storage/cdb_map.go
-
13go/storage/needle_map.go
-
8go/storage/store.go
-
9go/topology/topology.go
-
3go/weed/master.go
-
3go/weed/server.go
-
7go/weed/weed_server/master_server.go
-
3go/weed/weed_server/master_server_handlers_admin.go
-
20go/weed/weed_server/raft_server.go
-
2go/weed/weed_server/raft_server_handlers.go
-
4go/weed/weed_server/volume_server.go
@ -1,30 +0,0 @@ |
|||
package metastore |
|||
|
|||
import ( |
|||
"testing" |
|||
) |
|||
|
|||
func TestMemoryBacking(t *testing.T) { |
|||
ms := &MetaStore{NewMetaStoreMemoryBacking()} |
|||
verifySetGet(t, ms) |
|||
} |
|||
|
|||
func TestFileBacking(t *testing.T) { |
|||
ms := &MetaStore{NewMetaStoreFileBacking()} |
|||
verifySetGet(t, ms) |
|||
} |
|||
|
|||
func verifySetGet(t *testing.T, ms *MetaStore) { |
|||
data := uint64(234234) |
|||
ms.SetUint64("/tmp/sequence", data) |
|||
if !ms.Has("/tmp/sequence") { |
|||
t.Errorf("Failed to set data") |
|||
} |
|||
if val, err := ms.GetUint64("/tmp/sequence"); err == nil { |
|||
if val != data { |
|||
t.Errorf("Set %d, but read back %d", data, val) |
|||
} |
|||
} else { |
|||
t.Errorf("Failed to get back data:%s", err) |
|||
} |
|||
} |
|||
@ -1,34 +0,0 @@ |
|||
package metastore |
|||
|
|||
import ( |
|||
"io/ioutil" |
|||
"os" |
|||
) |
|||
|
|||
// store data on disk, enough for most cases
|
|||
|
|||
type MetaStoreFileBacking struct { |
|||
} |
|||
|
|||
func NewMetaStoreFileBacking() *MetaStoreFileBacking { |
|||
mms := &MetaStoreFileBacking{} |
|||
return mms |
|||
} |
|||
|
|||
func (mms *MetaStoreFileBacking) Set(path, val string) error { |
|||
return ioutil.WriteFile(path, []byte(val), 0644) |
|||
} |
|||
|
|||
func (mms *MetaStoreFileBacking) Get(path string) (string, error) { |
|||
val, e := ioutil.ReadFile(path) |
|||
return string(val), e |
|||
} |
|||
|
|||
func (mms *MetaStoreFileBacking) Has(path string) (ok bool) { |
|||
seqFile, se := os.OpenFile(path, os.O_RDONLY, 0644) |
|||
if se != nil { |
|||
return false |
|||
} |
|||
defer seqFile.Close() |
|||
return true |
|||
} |
|||
@ -1,36 +0,0 @@ |
|||
package metastore |
|||
|
|||
import ( |
|||
"fmt" |
|||
) |
|||
|
|||
//this is for testing only
|
|||
|
|||
type MetaStoreMemoryBacking struct { |
|||
m map[string]string |
|||
} |
|||
|
|||
func NewMetaStoreMemoryBacking() *MetaStoreMemoryBacking { |
|||
mms := &MetaStoreMemoryBacking{} |
|||
mms.m = make(map[string]string) |
|||
return mms |
|||
} |
|||
|
|||
func (mms MetaStoreMemoryBacking) Set(path, val string) error { |
|||
mms.m[path] = val |
|||
return nil |
|||
} |
|||
|
|||
func (mms MetaStoreMemoryBacking) Get(path string) (val string, err error) { |
|||
var ok bool |
|||
val, ok = mms.m[path] |
|||
if !ok { |
|||
return "", fmt.Errorf("Missing value for %s", path) |
|||
} |
|||
return |
|||
} |
|||
|
|||
func (mms MetaStoreMemoryBacking) Has(path string) (ok bool) { |
|||
_, ok = mms.m[path] |
|||
return |
|||
} |
|||
@ -1,33 +0,0 @@ |
|||
package metastore |
|||
|
|||
import ( |
|||
"errors" |
|||
"strconv" |
|||
) |
|||
|
|||
type MetaStoreBacking interface { |
|||
Get(path string) (string, error) |
|||
Set(path, val string) error |
|||
Has(path string) bool |
|||
} |
|||
|
|||
type MetaStore struct { |
|||
MetaStoreBacking |
|||
} |
|||
|
|||
func (m *MetaStore) SetUint64(path string, val uint64) error { |
|||
return m.Set(path, strconv.FormatUint(val, 10)) |
|||
} |
|||
|
|||
func (m *MetaStore) GetUint64(path string) (val uint64, err error) { |
|||
if b, e := m.Get(path); e == nil { |
|||
val, err = strconv.ParseUint(b, 10, 64) |
|||
return |
|||
} else { |
|||
if e != nil { |
|||
return 0, e |
|||
} |
|||
err = errors.New("Not found value for " + path) |
|||
} |
|||
return |
|||
} |
|||
@ -1,89 +1,9 @@ |
|||
package sequence |
|||
|
|||
import ( |
|||
"bytes" |
|||
"code.google.com/p/weed-fs/go/glog" |
|||
"code.google.com/p/weed-fs/go/metastore" |
|||
"encoding/gob" |
|||
"sync" |
|||
) |
|||
|
|||
const ( |
|||
FileIdSaveInterval = 10000 |
|||
) |
|||
import () |
|||
|
|||
type Sequencer interface { |
|||
NextFileId(count int) (uint64, int) |
|||
} |
|||
type SequencerImpl struct { |
|||
fileFullPath string |
|||
|
|||
volumeLock sync.Mutex |
|||
sequenceLock sync.Mutex |
|||
|
|||
FileIdSequence uint64 |
|||
fileIdCounter uint64 |
|||
|
|||
metaStore *metastore.MetaStore |
|||
} |
|||
|
|||
func NewFileSequencer(filepath string) (m *SequencerImpl) { |
|||
m = &SequencerImpl{fileFullPath: filepath} |
|||
m.metaStore = &metastore.MetaStore{metastore.NewMetaStoreFileBacking()} |
|||
m.initilize() |
|||
return |
|||
} |
|||
|
|||
func (m *SequencerImpl) initilize() { |
|||
if !m.metaStore.Has(m.fileFullPath) { |
|||
m.FileIdSequence = FileIdSaveInterval |
|||
glog.V(0).Infoln("Setting file id sequence", m.FileIdSequence) |
|||
} else { |
|||
var err error |
|||
if m.FileIdSequence, err = m.metaStore.GetUint64(m.fileFullPath); err != nil { |
|||
if data, err := m.metaStore.Get(m.fileFullPath); err == nil { |
|||
m.FileIdSequence = decode(data) |
|||
glog.V(0).Infoln("Decoding old version of FileIdSequence", m.FileIdSequence) |
|||
} else { |
|||
glog.V(0).Infof("No existing FileIdSequence: %s", err) |
|||
} |
|||
} else { |
|||
glog.V(0).Infoln("Loading file id sequence", m.FileIdSequence) |
|||
} |
|||
//in case the server stops between intervals
|
|||
} |
|||
return |
|||
} |
|||
|
|||
//count should be 1 or more
|
|||
func (m *SequencerImpl) NextFileId(count int) (uint64, int) { |
|||
if count <= 0 { |
|||
return 0, 0 |
|||
} |
|||
m.sequenceLock.Lock() |
|||
defer m.sequenceLock.Unlock() |
|||
if m.fileIdCounter < uint64(count) { |
|||
m.fileIdCounter = FileIdSaveInterval |
|||
m.FileIdSequence += FileIdSaveInterval |
|||
m.saveSequence() |
|||
} |
|||
m.fileIdCounter = m.fileIdCounter - uint64(count) |
|||
return m.FileIdSequence - m.fileIdCounter - uint64(count), count |
|||
} |
|||
func (m *SequencerImpl) saveSequence() { |
|||
glog.V(0).Infoln("Saving file id sequence", m.FileIdSequence, "to", m.fileFullPath) |
|||
if e := m.metaStore.SetUint64(m.fileFullPath, m.FileIdSequence); e != nil { |
|||
glog.Fatalf("Sequence id Save [ERROR] %s", e) |
|||
} |
|||
} |
|||
|
|||
//decode are for backward compatible purpose
|
|||
func decode(input string) uint64 { |
|||
var x uint64 |
|||
b := bytes.NewReader([]byte(input)) |
|||
decoder := gob.NewDecoder(b) |
|||
if e := decoder.Decode(&x); e == nil { |
|||
return x |
|||
} |
|||
return 0 |
|||
SetMax(uint64) |
|||
Peek() uint64 |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue