Chris Lu
3 years ago
11 changed files with 295 additions and 47 deletions
-
1weed/command/imports.go
-
5weed/filer.toml
-
88weed/filer/redis3/kv_directory_children.go
-
52weed/filer/redis3/skiplist_element_store.go
-
34weed/filer/redis3/universal_redis_store.go
-
1weed/server/filer_server.go
-
2weed/util/skiplist/name_batch.go
-
25weed/util/skiplist/name_list.go
-
71weed/util/skiplist/name_list_serde.go
-
4weed/util/skiplist/name_list_test.go
-
59weed/util/skiplist/skiplist.go
@ -0,0 +1,5 @@ |
|||||
|
[redis3] |
||||
|
enabled = true |
||||
|
address = "localhost:6379" |
||||
|
password = "" |
||||
|
database = 0 |
@ -0,0 +1,52 @@ |
|||||
|
package redis3 |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util/skiplist" |
||||
|
"github.com/go-redis/redis/v8" |
||||
|
"github.com/golang/protobuf/proto" |
||||
|
) |
||||
|
|
||||
|
type SkipListElementStore struct { |
||||
|
prefix string |
||||
|
client redis.UniversalClient |
||||
|
} |
||||
|
|
||||
|
var _ = skiplist.ListStore(&SkipListElementStore{}) |
||||
|
|
||||
|
func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore { |
||||
|
return &SkipListElementStore{ |
||||
|
prefix: prefix, |
||||
|
client: client, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error { |
||||
|
key := fmt.Sprintf("%s%d", m.prefix, id) |
||||
|
data, err := proto.Marshal(element) |
||||
|
if err != nil { |
||||
|
glog.Errorf("marshal %s: %v", key, err) |
||||
|
} |
||||
|
return m.client.Set(context.Background(), key, data, 0).Err() |
||||
|
} |
||||
|
|
||||
|
func (m *SkipListElementStore) DeleteElement(id int64) error { |
||||
|
key := fmt.Sprintf("%s%d", m.prefix, id) |
||||
|
return m.client.Del(context.Background(), key).Err() |
||||
|
} |
||||
|
|
||||
|
func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) { |
||||
|
key := fmt.Sprintf("%s%d", m.prefix, id) |
||||
|
data, err := m.client.Get(context.Background(), key).Result() |
||||
|
if err != nil { |
||||
|
if err == redis.Nil { |
||||
|
return nil, nil |
||||
|
} |
||||
|
return nil, err |
||||
|
} |
||||
|
t := &skiplist.SkipListElement{} |
||||
|
err = proto.Unmarshal([]byte(data), t) |
||||
|
return t, err |
||||
|
} |
@ -0,0 +1,71 @@ |
|||||
|
package skiplist |
||||
|
|
||||
|
import ( |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/golang/protobuf/proto" |
||||
|
) |
||||
|
|
||||
|
func LoadNameList(data []byte, store ListStore, batchSize int) *NameList { |
||||
|
|
||||
|
nl := &NameList{ |
||||
|
skipList: New(store), |
||||
|
batchSize: batchSize, |
||||
|
} |
||||
|
|
||||
|
if len(data) == 0 { |
||||
|
return nl |
||||
|
} |
||||
|
|
||||
|
message := &SkipListProto{} |
||||
|
if err := proto.Unmarshal(data, message); err != nil { |
||||
|
glog.Errorf("loading skiplist: %v", err) |
||||
|
} |
||||
|
nl.skipList.maxNewLevel = int(message.MaxNewLevel) |
||||
|
nl.skipList.maxLevel = int(message.MaxLevel) |
||||
|
for i, ref := range message.StartLevels { |
||||
|
nl.skipList.startLevels[i] = &SkipListElementReference{ |
||||
|
ElementPointer: ref.ElementPointer, |
||||
|
Key: ref.Key, |
||||
|
} |
||||
|
} |
||||
|
for i, ref := range message.EndLevels { |
||||
|
nl.skipList.endLevels[i] = &SkipListElementReference{ |
||||
|
ElementPointer: ref.ElementPointer, |
||||
|
Key: ref.Key, |
||||
|
} |
||||
|
} |
||||
|
return nl |
||||
|
} |
||||
|
|
||||
|
func (nl *NameList) HasChanges() bool { |
||||
|
return nl.skipList.hasChanges |
||||
|
} |
||||
|
|
||||
|
func (nl *NameList) ToBytes() []byte { |
||||
|
message := &SkipListProto{} |
||||
|
message.MaxNewLevel = int32(nl.skipList.maxNewLevel) |
||||
|
message.MaxLevel = int32(nl.skipList.maxLevel) |
||||
|
for _, ref := range nl.skipList.startLevels { |
||||
|
if ref == nil { |
||||
|
break |
||||
|
} |
||||
|
message.StartLevels = append(message.StartLevels, &SkipListElementReference{ |
||||
|
ElementPointer: ref.ElementPointer, |
||||
|
Key: ref.Key, |
||||
|
}) |
||||
|
} |
||||
|
for _, ref := range nl.skipList.endLevels { |
||||
|
if ref == nil { |
||||
|
break |
||||
|
} |
||||
|
message.EndLevels = append(message.EndLevels, &SkipListElementReference{ |
||||
|
ElementPointer: ref.ElementPointer, |
||||
|
Key: ref.Key, |
||||
|
}) |
||||
|
} |
||||
|
data, err := proto.Marshal(message) |
||||
|
if err != nil { |
||||
|
glog.Errorf("marshal skiplist: %v", err) |
||||
|
} |
||||
|
return data |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue