chrislu
10 months ago
18 changed files with 280 additions and 99 deletions
-
2.github/workflows/container_dev.yml
-
2.github/workflows/container_latest.yml
-
2.github/workflows/container_release1.yml
-
2.github/workflows/container_release2.yml
-
2.github/workflows/container_release3.yml
-
2.github/workflows/container_release4.yml
-
2.github/workflows/container_release5.yml
-
2.github/workflows/helm_ci.yml
-
3docker/Makefile
-
54docker/compose/local-filer-backup-compose.yml
-
16go.mod
-
38go.sum
-
3weed/command/filer_backup.go
-
121weed/replication/sink/s3sink/s3_sink.go
-
1weed/s3api/s3_constants/header.go
-
67weed/s3api/s3api_objects_list_handlers.go
-
53weed/shell/command_fs_log.go
-
7weed/storage/store.go
@ -0,0 +1,54 @@ |
|||
version: '3.9' |
|||
|
|||
services: |
|||
server-left: |
|||
image: chrislusf/seaweedfs:local |
|||
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" |
|||
volumes: |
|||
- ./s3.json:/etc/seaweedfs/s3.json |
|||
healthcheck: |
|||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] |
|||
interval: 3s |
|||
start_period: 15s |
|||
timeout: 30s |
|||
server-right: |
|||
image: chrislusf/seaweedfs:local |
|||
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" |
|||
volumes: |
|||
- ./s3.json:/etc/seaweedfs/s3.json |
|||
healthcheck: |
|||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] |
|||
interval: 3s |
|||
start_period: 15s |
|||
timeout: 30s |
|||
filer-backup: |
|||
image: chrislusf/seaweedfs:local |
|||
command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888" |
|||
volumes: |
|||
- ./replication.toml:/etc/seaweedfs/replication.toml |
|||
environment: |
|||
WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false" |
|||
WEED_SINK_S3_ENABLED: "true" |
|||
WEED_SINK_S3_BUCKET: "backup" |
|||
WEED_SINK_S3_ENDPOINT: "http://server-right:8333" |
|||
WEED_SINK_S3_DIRECTORY: "/" |
|||
WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1" |
|||
WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1" |
|||
WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false" |
|||
WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5" |
|||
WEED_SINK_S3_KEEP_PART_SIZE: "false" |
|||
depends_on: |
|||
server-left: |
|||
condition: service_healthy |
|||
server-right: |
|||
condition: service_healthy |
|||
minio-warp: |
|||
image: minio/warp |
|||
command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2' |
|||
restart: on-failure |
|||
environment: |
|||
WARP_HOST: "server-left:8333" |
|||
WARP_ACCESS_KEY: "some_access_key1" |
|||
WARP_SECRET_KEY: "some_secret_key1" |
|||
depends_on: |
|||
- filer-backup |
|||
@ -0,0 +1,53 @@ |
|||
package shell |
|||
|
|||
import ( |
|||
"flag" |
|||
"fmt" |
|||
"github.com/seaweedfs/seaweedfs/weed/filer" |
|||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" |
|||
"io" |
|||
"time" |
|||
) |
|||
|
|||
func init() { |
|||
Commands = append(Commands, &commandFsLogPurge{}) |
|||
} |
|||
|
|||
type commandFsLogPurge struct { |
|||
} |
|||
|
|||
func (c *commandFsLogPurge) Name() string { |
|||
return "fs.log.purge" |
|||
} |
|||
|
|||
func (c *commandFsLogPurge) Help() string { |
|||
return `purge filer logs |
|||
|
|||
fs.log.purge [-v] [-modifyDayAgo 365] |
|||
` |
|||
} |
|||
|
|||
func (c *commandFsLogPurge) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { |
|||
fsLogPurgeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) |
|||
daysAgo := fsLogPurgeCommand.Uint("daysAgo", 365, "purge logs older than N days") |
|||
verbose := fsLogPurgeCommand.Bool("v", false, "verbose mode") |
|||
|
|||
if err = fsLogPurgeCommand.Parse(args); err != nil { |
|||
return err |
|||
} |
|||
|
|||
modificationTimeAgo := time.Now().Add(-time.Hour * 24 * time.Duration(*daysAgo)).Unix() |
|||
err = filer_pb.ReadDirAllEntries(commandEnv, filer.SystemLogDir, "", func(entry *filer_pb.Entry, isLast bool) error { |
|||
if entry.Attributes.Mtime > modificationTimeAgo { |
|||
return nil |
|||
} |
|||
if errDel := filer_pb.Remove(commandEnv, filer.SystemLogDir, entry.Name, true, true, true, false, nil); errDel != nil { |
|||
return errDel |
|||
} |
|||
if *verbose { |
|||
fmt.Fprintf(writer, "delete %s\n", entry.Name) |
|||
} |
|||
return nil |
|||
}) |
|||
return err |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue