Chris Lu
3 years ago
39 changed files with 1707 additions and 1112 deletions
-
8.github/workflows/binaries_dev.yml
-
2.github/workflows/container_dev.yml
-
59.github/workflows/container_release.yml
-
59docker/Dockerfile.rocksdb_large
-
6docker/Makefile
-
20go.mod
-
35go.sum
-
4k8s/helm_charts2/Chart.yaml
-
2k8s/helm_charts2/values.yaml
-
10test/s3/multipart/aws_upload.go
-
64unmaintained/stream_read_volume/stream_read_volume.go
-
7weed/command/filer.go
-
6weed/command/filer_meta_tail.go
-
6weed/command/master.go
-
6weed/command/master_follower.go
-
2weed/command/s3.go
-
8weed/command/server.go
-
5weed/command/volume.go
-
2weed/filer/filer.go
-
12weed/filer/filer_notify.go
-
2weed/filer/filer_notify_append.go
-
2weed/filer/meta_aggregator.go
-
2weed/filer/rocksdb/rocksdb_store.go
-
7weed/filer/rocksdb/rocksdb_ttl.go
-
6weed/messaging/broker/broker_grpc_server_subscribe.go
-
2weed/pb/server_address.go
-
12weed/pb/volume_server.proto
-
2206weed/pb/volume_server_pb/volume_server.pb.go
-
6weed/server/filer_grpc_server_sub_meta.go
-
36weed/server/volume_grpc_read_all.go
-
12weed/shell/command_ec_decode.go
-
119weed/shell/command_volume_fix_replication.go
-
2weed/storage/backend/s3_backend/s3_download.go
-
4weed/storage/needle/needle.go
-
42weed/storage/volume_read_all.go
-
2weed/topology/volume_layout.go
-
2weed/util/constants.go
-
8weed/util/file_util.go
-
24weed/util/log_buffer/log_buffer.go
@ -0,0 +1,59 @@ |
|||
FROM golang:1.17-buster as builder |
|||
|
|||
RUN apt-get update |
|||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev |
|||
|
|||
ENV ROCKSDB_VERSION v6.22.1 |
|||
|
|||
# build RocksDB |
|||
RUN cd /tmp && \ |
|||
git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ |
|||
cd rocksdb && \ |
|||
make static_lib && \ |
|||
make install-static |
|||
|
|||
ENV CGO_CFLAGS "-I/tmp/rocksdb/include" |
|||
ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" |
|||
|
|||
# build SeaweedFS |
|||
RUN mkdir -p /go/src/github.com/chrislusf/ |
|||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs |
|||
ARG BRANCH=${BRANCH:-master} |
|||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH |
|||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ |
|||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ |
|||
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" |
|||
|
|||
|
|||
FROM alpine AS final |
|||
LABEL author="Chris Lu" |
|||
COPY --from=builder /go/bin/weed /usr/bin/ |
|||
RUN mkdir -p /etc/seaweedfs |
|||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml |
|||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh |
|||
RUN apk add fuse snappy gflags |
|||
|
|||
# volume server gprc port |
|||
EXPOSE 18080 |
|||
# volume server http port |
|||
EXPOSE 8080 |
|||
# filer server gprc port |
|||
EXPOSE 18888 |
|||
# filer server http port |
|||
EXPOSE 8888 |
|||
# master server shared gprc port |
|||
EXPOSE 19333 |
|||
# master server shared http port |
|||
EXPOSE 9333 |
|||
# s3 server http port |
|||
EXPOSE 8333 |
|||
# webdav server http port |
|||
EXPOSE 7333 |
|||
|
|||
RUN mkdir -p /data/filerldb2 |
|||
|
|||
VOLUME /data |
|||
|
|||
RUN chmod +x /entrypoint.sh |
|||
|
|||
ENTRYPOINT ["/entrypoint.sh"] |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.69" |
|||
version: "2.69" |
|||
appVersion: "2.70" |
|||
version: "2.70" |
|||
@ -0,0 +1,64 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"context" |
|||
"errors" |
|||
"flag" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/operation" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"google.golang.org/grpc" |
|||
"io" |
|||
) |
|||
|
|||
var ( |
|||
volumeServer = flag.String("volumeServer", "localhost:8080", "a volume server") |
|||
volumeId = flag.Int("volumeId", -1, "a volume id to stream read") |
|||
grpcDialOption grpc.DialOption |
|||
) |
|||
|
|||
func main() { |
|||
flag.Parse() |
|||
|
|||
util.LoadConfiguration("security", false) |
|||
grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") |
|||
|
|||
vid := uint32(*volumeId) |
|||
|
|||
eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error { |
|||
fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob)) |
|||
return nil |
|||
} |
|||
|
|||
err := operation.WithVolumeServerClient(pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error { |
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
defer cancel() |
|||
copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{ |
|||
VolumeIds: []uint32{vid}, |
|||
}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
for { |
|||
resp, err := copyFileClient.Recv() |
|||
if errors.Is(err, io.EOF) { |
|||
break |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if err = eachNeedleFunc(resp); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
}) |
|||
if err != nil { |
|||
fmt.Printf("read %s: %v\n", *volumeServer, err) |
|||
} |
|||
|
|||
} |
|||
|
|||
2206
weed/pb/volume_server_pb/volume_server.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
@ -0,0 +1,36 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
) |
|||
|
|||
func (vs *VolumeServer) ReadAllNeedles(req *volume_server_pb.ReadAllNeedlesRequest, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer) (err error) { |
|||
|
|||
for _, vid := range req.VolumeIds { |
|||
if err := vs.streaReadOneVolume(needle.VolumeId(vid), stream, err); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (vs *VolumeServer) streaReadOneVolume(vid needle.VolumeId, stream volume_server_pb.VolumeServer_ReadAllNeedlesServer, err error) error { |
|||
v := vs.store.GetVolume(vid) |
|||
if v == nil { |
|||
return fmt.Errorf("not found volume id %d", vid) |
|||
} |
|||
|
|||
scanner := &storage.VolumeFileScanner4ReadAll{ |
|||
Stream: stream, |
|||
V: v, |
|||
} |
|||
|
|||
offset := int64(v.SuperBlock.BlockSize()) |
|||
|
|||
err = storage.ScanVolumeFileFrom(v.Version(), v.DataBackend, offset, scanner) |
|||
|
|||
return err |
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/super_block" |
|||
) |
|||
|
|||
type VolumeFileScanner4ReadAll struct { |
|||
Stream volume_server_pb.VolumeServer_ReadAllNeedlesServer |
|||
V *Volume |
|||
} |
|||
|
|||
func (scanner *VolumeFileScanner4ReadAll) VisitSuperBlock(superBlock super_block.SuperBlock) error { |
|||
return nil |
|||
|
|||
} |
|||
func (scanner *VolumeFileScanner4ReadAll) ReadNeedleBody() bool { |
|||
return true |
|||
} |
|||
|
|||
func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error { |
|||
|
|||
nv, ok := scanner.V.nm.Get(n.Id) |
|||
if !ok { |
|||
return nil |
|||
} |
|||
if nv.Offset.ToActualOffset() != offset { |
|||
return nil |
|||
} |
|||
|
|||
sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{ |
|||
VolumeId: uint32(scanner.V.Id), |
|||
NeedleId: uint64(n.Id), |
|||
Cookie: uint32(n.Cookie), |
|||
NeedleBlob: n.Data, |
|||
}) |
|||
if sendErr != nil { |
|||
return sendErr |
|||
} |
|||
return nil |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue