From 6d2fda27d21d98aa73b072bd48ab0c3bf7c76281 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 25 Apr 2022 22:59:46 +0500 Subject: [PATCH 1/3] delete missing data from volumes in one replica --- weed/shell/command_volume_fsck.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 1aa33e054..e950c0b0e 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -33,7 +33,8 @@ func init() { } type commandVolumeFsck struct { - env *CommandEnv + env *CommandEnv + forcePurging bool } func (c *commandVolumeFsck) Name() string { @@ -68,6 +69,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") + c.forcePurging = *fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") @@ -293,7 +295,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn } orphanFileIds := []string{} for fid, foundInAllReplicas := range orphanReplicaFileIds { - if !isSeveralReplicas[volumeId] || (isSeveralReplicas[volumeId] && foundInAllReplicas) { + if !isSeveralReplicas[volumeId] || c.forcePurging || (isSeveralReplicas[volumeId] && foundInAllReplicas) { orphanFileIds = append(orphanFileIds, fid) } } @@ -301,7 +303,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn continue } if verbose { - fmt.Fprintf(writer, "purging process for volume %d", volumeId) + fmt.Fprintf(writer, "purging process for volume %d.\n", volumeId) } if isEcVolumeReplicas[volumeId] { From ae56b2c00f8db6cd5b6958b755c24aaf0e753215 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 25 Apr 2022 23:10:01 +0500 Subject: [PATCH 2/3] change forcePurging to a pointer --- weed/shell/command_volume_fsck.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index e950c0b0e..c79b062f7 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -34,7 +34,7 @@ func init() { type commandVolumeFsck struct { env *CommandEnv - forcePurging bool + forcePurging *bool } func (c *commandVolumeFsck) Name() string { @@ -69,7 +69,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") - c.forcePurging = *fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica") + c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") @@ -295,7 +295,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn } orphanFileIds := []string{} for fid, foundInAllReplicas := range orphanReplicaFileIds { - if !isSeveralReplicas[volumeId] || c.forcePurging || (isSeveralReplicas[volumeId] && foundInAllReplicas) { + if !isSeveralReplicas[volumeId] || *c.forcePurging || (isSeveralReplicas[volumeId] && foundInAllReplicas) { orphanFileIds = append(orphanFileIds, fid) } } From d4343ab7da4171088ab328d9d688885b67fd1d0c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 25 Apr 2022 23:11:56 +0500 Subject: [PATCH 3/3] forcePurging desc --- weed/shell/command_volume_fsck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index c79b062f7..584ce722b 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -69,7 +69,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, " after detection, delete missing data from volumes / delete missing file entries from filer") - c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica") + c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, " delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files")