Browse Source

filer: etcd store fix listing

fix https://github.com/chrislusf/seaweedfs/issues/1767
pull/1778/head
Chris Lu 4 years ago
parent
commit
314dc1c957
  1. 3
      docker/Makefile
  2. 84
      docker/swarm-etcd.yml
  3. 12
      weed/filer/etcd/etcd_store.go

3
docker/Makefile

@ -30,5 +30,8 @@ cluster: build
2clusters: build 2clusters: build
docker-compose -f local-clusters-compose.yml -p seaweedfs up docker-compose -f local-clusters-compose.yml -p seaweedfs up
filer_etcd: build
docker stack deploy -c swarm-etcd.yml fs
clean: clean:
rm ./weed rm ./weed

84
docker/swarm-etcd.yml

@ -0,0 +1,84 @@
# 2021-01-30 16:25:30
version: '3.8'
services:
etcd:
image: gasparekatapy/etcd
networks:
- net
deploy:
mode: replicated
replicas: 3
master:
image: chrislusf/seaweedfs:local
environment:
WEED_MASTER_FILER_DEFAULT: "filer:8888"
WEED_MASTER_SEQUENCER_TYPE: "raft"
ports:
- "9333:9333"
- "19333:19333"
networks:
- net
command:
- 'master'
- '-resumeState=true'
- '-ip=master'
- '-port=9333'
deploy:
mode: replicated
replicas: 1
filer:
image: chrislusf/seaweedfs:local
environment:
WEED_LEVELDB2_ENABLED: "false"
WEED_ETCD_ENABLED: "true"
WEED_ETCD_SERVERS: "etcd:2379"
ports:
- target: 8888
published: 8888
protocol: tcp
mode: host
- target: 18888
published: 18888
protocol: tcp
mode: host
networks:
- net
command:
- 'filer'
- '-ip=filer'
- '-port=8888'
- '-port.readonly=28888'
- '-master=master:9333'
- '-disableDirListing=true'
deploy:
mode: replicated
replicas: 1
volume:
image: chrislusf/seaweedfs:local
ports:
- target: 8080
published: 8080
protocol: tcp
mode: host
- target: 18080
published: 18080
protocol: tcp
mode: host
networks:
- net
command:
- 'volume'
- '-mserver=master:9333'
- '-port=8080'
deploy:
mode: global
###########################################################################
networks:
net:

12
weed/filer/etcd/etcd_store.go

@ -1,6 +1,7 @@
package etcd package etcd
import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"strings" "strings"
@ -145,14 +146,21 @@ func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPat
func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
directoryPrefix := genDirectoryKeyPrefix(dirPath, "") directoryPrefix := genDirectoryKeyPrefix(dirPath, "")
lastFileStart := directoryPrefix
if startFileName != "" {
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
}
resp, err := store.client.Get(ctx, string(directoryPrefix),
resp, err := store.client.Get(ctx, string(lastFileStart),
clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend))
if err != nil { if err != nil {
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
} }
for _, kv := range resp.Kvs { for _, kv := range resp.Kvs {
if !bytes.HasPrefix(kv.Key, directoryPrefix) {
break
}
fileName := getNameFromKey(kv.Key) fileName := getNameFromKey(kv.Key)
if fileName == "" { if fileName == "" {
continue continue
@ -160,7 +168,6 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
if fileName == startFileName && !includeStartFile { if fileName == startFileName && !includeStartFile {
continue continue
} }
lastFileName = fileName
limit-- limit--
if limit < 0 { if limit < 0 {
break break
@ -176,6 +183,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
if !eachEntryFunc(entry) { if !eachEntryFunc(entry) {
break break
} }
lastFileName = fileName
} }
return lastFileName, err return lastFileName, err

Loading…
Cancel
Save