chrislu
6 months ago
10 changed files with 865 additions and 486 deletions
-
12other/java/client/src/main/proto/filer.proto
-
26weed/filer/filer.go
-
12weed/pb/filer.proto
-
1052weed/pb/filer_pb/filer.pb.go
-
36weed/pb/filer_pb/filer_client_bfs.go
-
68weed/pb/filer_pb/filer_grpc.pb.go
-
84weed/server/filer_grpc_server_traverse_meta.go
-
31weed/server/filer_grpc_server_traverse_meta_test.go
-
2weed/server/filer_server.go
-
30weed/util/queue.go
1052
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,84 @@ |
|||||
|
package weed_server |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/filer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
"github.com/viant/ptrie" |
||||
|
) |
||||
|
|
||||
|
func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequest, stream filer_pb.SeaweedFiler_TraverseBfsMetadataServer) error { |
||||
|
|
||||
|
glog.V(0).Infof("TraverseBfsMetadata %v", req) |
||||
|
|
||||
|
excludedTrie := ptrie.New[bool]() |
||||
|
for _, excluded := range req.ExcludedPrefixes { |
||||
|
excludedTrie.Put([]byte(excluded), true) |
||||
|
} |
||||
|
|
||||
|
ctx := stream.Context() |
||||
|
|
||||
|
queue := util.NewQueue[*filer.Entry]() |
||||
|
dirEntry, err := fs.filer.FindEntry(ctx, util.FullPath(req.Directory)) |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("find dir %s: %v", req.Directory, err) |
||||
|
} |
||||
|
queue.Enqueue(dirEntry) |
||||
|
|
||||
|
for item := queue.Dequeue(); item != nil; item = queue.Dequeue() { |
||||
|
if excludedTrie.MatchPrefix([]byte(item.FullPath), func(key []byte, value bool) bool { |
||||
|
return true |
||||
|
}) { |
||||
|
// println("excluded", item.FullPath)
|
||||
|
continue |
||||
|
} |
||||
|
parent, _ := item.FullPath.DirAndName() |
||||
|
if err := stream.Send(&filer_pb.TraverseBfsMetadataResponse{ |
||||
|
Directory: parent, |
||||
|
Entry: item.ToProtoEntry(), |
||||
|
}); err != nil { |
||||
|
return fmt.Errorf("send traverse bfs metadata response: %v", err) |
||||
|
} |
||||
|
|
||||
|
if !item.IsDirectory() { |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
if err := fs.iterateDirectory(ctx, item.FullPath, func(entry *filer.Entry) error { |
||||
|
queue.Enqueue(entry) |
||||
|
return nil |
||||
|
}); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (fs *FilerServer) iterateDirectory(ctx context.Context, dirPath util.FullPath, fn func(entry *filer.Entry) error) (err error) { |
||||
|
var lastFileName string |
||||
|
var listErr error |
||||
|
for { |
||||
|
var hasEntries bool |
||||
|
lastFileName, listErr = fs.filer.StreamListDirectoryEntries(ctx, dirPath, lastFileName, false, 1024, "", "", "", func(entry *filer.Entry) bool { |
||||
|
hasEntries = true |
||||
|
if fnErr := fn(entry); fnErr != nil { |
||||
|
err = fnErr |
||||
|
return false |
||||
|
} |
||||
|
return true |
||||
|
}) |
||||
|
if listErr != nil { |
||||
|
return listErr |
||||
|
} |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
if !hasEntries { |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
} |
@ -0,0 +1,31 @@ |
|||||
|
package weed_server |
||||
|
|
||||
|
import ( |
||||
|
"github.com/stretchr/testify/assert" |
||||
|
"github.com/viant/ptrie" |
||||
|
"testing" |
||||
|
) |
||||
|
|
||||
|
func TestPtrie(t *testing.T) { |
||||
|
b := []byte("/topics/abc/dev") |
||||
|
excludedTrie := ptrie.New[bool]() |
||||
|
excludedTrie.Put([]byte("/topics/abc/d"), true) |
||||
|
excludedTrie.Put([]byte("/topics/abc"), true) |
||||
|
|
||||
|
assert.True(t, excludedTrie.MatchPrefix(b, func(key []byte, value bool) bool { |
||||
|
println("matched1", string(key)) |
||||
|
return true |
||||
|
})) |
||||
|
|
||||
|
assert.True(t, excludedTrie.MatchAll(b, func(key []byte, value bool) bool { |
||||
|
println("matched2", string(key)) |
||||
|
return true |
||||
|
})) |
||||
|
|
||||
|
assert.False(t, excludedTrie.MatchAll([]byte("/topics/ab"), func(key []byte, value bool) bool { |
||||
|
println("matched3", string(key)) |
||||
|
return true |
||||
|
})) |
||||
|
|
||||
|
assert.False(t, excludedTrie.Has(b)) |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue