Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Also delete paths for which a volume is entirely absent.

pull/2640/head
Radtoo 3 years ago
parent
fa0cfdfb7f
commit
6b17f45da2
1 changed files with 70 additions and 73 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 55
      weed/shell/command_volume_fsck.go

55
weed/shell/command_volume_fsck.go
View File

@ -101,11 +101,11 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
if *findMissingChunksInFiler { if *findMissingChunksInFiler {
// collect all filer file ids and paths // collect all filer file ids and paths
if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose); err != nil {
if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *applyPurging); err != nil {
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
} }
// for each volume, check filer file ids // for each volume, check filer file ids
if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil {
return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err)
} }
} else { } else {
@ -114,7 +114,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
return fmt.Errorf("failed to collect file ids from filer: %v", err) return fmt.Errorf("failed to collect file ids from filer: %v", err)
} }
// volume file ids subtract filer file ids // volume file ids subtract filer file ids
if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil {
return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) return fmt.Errorf("findExtraChunksInVolumeServers: %v", err)
} }
} }
@ -122,7 +122,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
return nil return nil
} }
func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool) error {
func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging bool) error {
if verbose { if verbose {
fmt.Fprintf(writer, "checking each file from filer ...\n") fmt.Fprintf(writer, "checking each file from filer ...\n")
@ -179,13 +179,17 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint
// fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path) // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path)
} else { } else {
fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
if applyPurging {
fmt.Printf("deleting path %s for volume not found", i.path)
c.httpDelete(i.path, verbose)
}
} }
} }
}) })
} }
func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error {
for volumeId, vinfo := range volumeIdToVInfo { for volumeId, vinfo := range volumeIdToVInfo {
checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging) checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging)
@ -196,7 +200,7 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf
return nil return nil
} }
func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error {
var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64
@ -215,9 +219,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u
} }
} }
if len(orphanFileIds) > 0 {
if *applyPurging {
if applyPurging && len(orphanFileIds) > 0 {
if verbose { if verbose {
fmt.Fprintf(writer, "purging process for volume %d", volumeId) fmt.Fprintf(writer, "purging process for volume %d", volumeId)
} }
@ -241,27 +243,17 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u
fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server)
// FIXME Does not delete all replicas of this volumeId on all volume servers. Using fileId based deletion in all cases for now.
// if inUseCount < 1 {
// if verbose {
// fmt.Fprintf(writer, "removing empty (all content orphaned) volume %d\n", volumeId)
// }
// if err := deleteVolume(c.env.option.GrpcDialOption, needleVID, vinfo.server); err != nil {
// return fmt.Errorf("removing empty volume %d: %v", volumeId, err)
// }
// } else {
if verbose { if verbose {
fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) fmt.Fprintf(writer, "purging files from volume %d\n", volumeId)
} }
if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil {
return fmt.Errorf("purging volume %d: %v", volumeId, err) return fmt.Errorf("purging volume %d: %v", volumeId, err)
} }
} }
} }
}
if !*applyPurging {
if !applyPurging {
pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount)) pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize) totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)
@ -364,7 +356,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInf
}) })
} }
func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging *bool) (err error) {
func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) {
if verbose { if verbose {
fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId)
@ -417,13 +409,22 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo
if _, found := db.Get(needleId); !found { if _, found := db.Get(needleId); !found {
fmt.Fprintf(writer, "%s\n", item.path) fmt.Fprintf(writer, "%s\n", item.path)
if *applyPurging {
req, err := http.NewRequest(http.MethodDelete, "", nil)
if applyPurging {
// defining the URL this way automatically escapes complex path names // defining the URL this way automatically escapes complex path names
c.httpDelete(item.path, verbose)
}
}
}
return nil
}
func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) {
req, err := http.NewRequest(http.MethodDelete, "", nil)
req.URL = &url.URL{ req.URL = &url.URL{
Scheme: "http", Scheme: "http",
Host: c.env.option.FilerAddress.ToHttpAddress(), Host: c.env.option.FilerAddress.ToHttpAddress(),
Path: string(item.path),
Path: string(path),
} }
if verbose { if verbose {
fmt.Printf("full HTTP delete request to be sent: %v\n", req) fmt.Printf("full HTTP delete request to be sent: %v\n", req)
@ -449,10 +450,6 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo
fmt.Println("delete response Status : ", resp.Status) fmt.Println("delete response Status : ", resp.Status)
fmt.Println("delete response Headers : ", resp.Header) fmt.Println("delete response Headers : ", resp.Header)
} }
}
}
}
return nil
} }
func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) {

Write Preview
Loading…
Cancel
Save