Chris Lu
3 years ago
11 changed files with 295 additions and 47 deletions
-
1weed/command/imports.go
-
5weed/filer.toml
-
88weed/filer/redis3/kv_directory_children.go
-
52weed/filer/redis3/skiplist_element_store.go
-
34weed/filer/redis3/universal_redis_store.go
-
1weed/server/filer_server.go
-
2weed/util/skiplist/name_batch.go
-
25weed/util/skiplist/name_list.go
-
71weed/util/skiplist/name_list_serde.go
-
4weed/util/skiplist/name_list_test.go
-
59weed/util/skiplist/skiplist.go
@ -0,0 +1,5 @@ |
|||
[redis3] |
|||
enabled = true |
|||
address = "localhost:6379" |
|||
password = "" |
|||
database = 0 |
@ -0,0 +1,52 @@ |
|||
package redis3 |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/util/skiplist" |
|||
"github.com/go-redis/redis/v8" |
|||
"github.com/golang/protobuf/proto" |
|||
) |
|||
|
|||
type SkipListElementStore struct { |
|||
prefix string |
|||
client redis.UniversalClient |
|||
} |
|||
|
|||
var _ = skiplist.ListStore(&SkipListElementStore{}) |
|||
|
|||
func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore { |
|||
return &SkipListElementStore{ |
|||
prefix: prefix, |
|||
client: client, |
|||
} |
|||
} |
|||
|
|||
func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error { |
|||
key := fmt.Sprintf("%s%d", m.prefix, id) |
|||
data, err := proto.Marshal(element) |
|||
if err != nil { |
|||
glog.Errorf("marshal %s: %v", key, err) |
|||
} |
|||
return m.client.Set(context.Background(), key, data, 0).Err() |
|||
} |
|||
|
|||
func (m *SkipListElementStore) DeleteElement(id int64) error { |
|||
key := fmt.Sprintf("%s%d", m.prefix, id) |
|||
return m.client.Del(context.Background(), key).Err() |
|||
} |
|||
|
|||
func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) { |
|||
key := fmt.Sprintf("%s%d", m.prefix, id) |
|||
data, err := m.client.Get(context.Background(), key).Result() |
|||
if err != nil { |
|||
if err == redis.Nil { |
|||
return nil, nil |
|||
} |
|||
return nil, err |
|||
} |
|||
t := &skiplist.SkipListElement{} |
|||
err = proto.Unmarshal([]byte(data), t) |
|||
return t, err |
|||
} |
@ -0,0 +1,71 @@ |
|||
package skiplist |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/golang/protobuf/proto" |
|||
) |
|||
|
|||
func LoadNameList(data []byte, store ListStore, batchSize int) *NameList { |
|||
|
|||
nl := &NameList{ |
|||
skipList: New(store), |
|||
batchSize: batchSize, |
|||
} |
|||
|
|||
if len(data) == 0 { |
|||
return nl |
|||
} |
|||
|
|||
message := &SkipListProto{} |
|||
if err := proto.Unmarshal(data, message); err != nil { |
|||
glog.Errorf("loading skiplist: %v", err) |
|||
} |
|||
nl.skipList.maxNewLevel = int(message.MaxNewLevel) |
|||
nl.skipList.maxLevel = int(message.MaxLevel) |
|||
for i, ref := range message.StartLevels { |
|||
nl.skipList.startLevels[i] = &SkipListElementReference{ |
|||
ElementPointer: ref.ElementPointer, |
|||
Key: ref.Key, |
|||
} |
|||
} |
|||
for i, ref := range message.EndLevels { |
|||
nl.skipList.endLevels[i] = &SkipListElementReference{ |
|||
ElementPointer: ref.ElementPointer, |
|||
Key: ref.Key, |
|||
} |
|||
} |
|||
return nl |
|||
} |
|||
|
|||
func (nl *NameList) HasChanges() bool { |
|||
return nl.skipList.hasChanges |
|||
} |
|||
|
|||
func (nl *NameList) ToBytes() []byte { |
|||
message := &SkipListProto{} |
|||
message.MaxNewLevel = int32(nl.skipList.maxNewLevel) |
|||
message.MaxLevel = int32(nl.skipList.maxLevel) |
|||
for _, ref := range nl.skipList.startLevels { |
|||
if ref == nil { |
|||
break |
|||
} |
|||
message.StartLevels = append(message.StartLevels, &SkipListElementReference{ |
|||
ElementPointer: ref.ElementPointer, |
|||
Key: ref.Key, |
|||
}) |
|||
} |
|||
for _, ref := range nl.skipList.endLevels { |
|||
if ref == nil { |
|||
break |
|||
} |
|||
message.EndLevels = append(message.EndLevels, &SkipListElementReference{ |
|||
ElementPointer: ref.ElementPointer, |
|||
Key: ref.Key, |
|||
}) |
|||
} |
|||
data, err := proto.Marshal(message) |
|||
if err != nil { |
|||
glog.Errorf("marshal skiplist: %v", err) |
|||
} |
|||
return data |
|||
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue