|
|
@ -101,11 +101,11 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. |
|
|
|
|
|
|
|
if *findMissingChunksInFiler { |
|
|
|
// collect all filer file ids and paths
|
|
|
|
if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose); err != nil { |
|
|
|
if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, *applyPurging); err != nil { |
|
|
|
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err) |
|
|
|
} |
|
|
|
// for each volume, check filer file ids
|
|
|
|
if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { |
|
|
|
if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { |
|
|
|
return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err) |
|
|
|
} |
|
|
|
} else { |
|
|
@ -114,7 +114,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. |
|
|
|
return fmt.Errorf("failed to collect file ids from filer: %v", err) |
|
|
|
} |
|
|
|
// volume file ids subtract filer file ids
|
|
|
|
if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil { |
|
|
|
if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, *applyPurging); err != nil { |
|
|
|
return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) |
|
|
|
} |
|
|
|
} |
|
|
@ -122,7 +122,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool) error { |
|
|
|
func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging bool) error { |
|
|
|
|
|
|
|
if verbose { |
|
|
|
fmt.Fprintf(writer, "checking each file from filer ...\n") |
|
|
@ -179,13 +179,17 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint |
|
|
|
// fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path)
|
|
|
|
} else { |
|
|
|
fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path) |
|
|
|
if applyPurging { |
|
|
|
fmt.Printf("deleting path %s for volume not found", i.path) |
|
|
|
c.httpDelete(i.path, verbose) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
}) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { |
|
|
|
func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { |
|
|
|
|
|
|
|
for volumeId, vinfo := range volumeIdToVInfo { |
|
|
|
checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose, applyPurging) |
|
|
@ -196,7 +200,7 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error { |
|
|
|
func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging bool) error { |
|
|
|
|
|
|
|
var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 |
|
|
|
|
|
|
@ -215,9 +219,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if len(orphanFileIds) > 0 { |
|
|
|
|
|
|
|
if *applyPurging { |
|
|
|
if applyPurging && len(orphanFileIds) > 0 { |
|
|
|
if verbose { |
|
|
|
fmt.Fprintf(writer, "purging process for volume %d", volumeId) |
|
|
|
} |
|
|
@ -241,27 +243,17 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u |
|
|
|
|
|
|
|
fmt.Fprintf(writer, "marked %d on server %v writable for forced purge\n", volumeId, vinfo.server) |
|
|
|
|
|
|
|
// FIXME Does not delete all replicas of this volumeId on all volume servers. Using fileId based deletion in all cases for now.
|
|
|
|
// if inUseCount < 1 {
|
|
|
|
// if verbose {
|
|
|
|
// fmt.Fprintf(writer, "removing empty (all content orphaned) volume %d\n", volumeId)
|
|
|
|
// }
|
|
|
|
// if err := deleteVolume(c.env.option.GrpcDialOption, needleVID, vinfo.server); err != nil {
|
|
|
|
// return fmt.Errorf("removing empty volume %d: %v", volumeId, err)
|
|
|
|
// }
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
if verbose { |
|
|
|
fmt.Fprintf(writer, "purging files from volume %d\n", volumeId) |
|
|
|
} |
|
|
|
|
|
|
|
if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil { |
|
|
|
return fmt.Errorf("purging volume %d: %v", volumeId, err) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if !*applyPurging { |
|
|
|
if !applyPurging { |
|
|
|
pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount)) |
|
|
|
fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n", |
|
|
|
totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize) |
|
|
@ -364,7 +356,7 @@ func (c *commandVolumeFsck) collectFilerFileIds(volumeIdToServer map[uint32]VInf |
|
|
|
}) |
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging *bool) (err error) { |
|
|
|
func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool, applyPurging bool) (err error) { |
|
|
|
|
|
|
|
if verbose { |
|
|
|
fmt.Fprintf(writer, "find missing file chunks in volume %d ...\n", volumeId) |
|
|
@ -417,13 +409,22 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo |
|
|
|
if _, found := db.Get(needleId); !found { |
|
|
|
fmt.Fprintf(writer, "%s\n", item.path) |
|
|
|
|
|
|
|
if *applyPurging { |
|
|
|
req, err := http.NewRequest(http.MethodDelete, "", nil) |
|
|
|
if applyPurging { |
|
|
|
// defining the URL this way automatically escapes complex path names
|
|
|
|
c.httpDelete(item.path, verbose) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) httpDelete(path util.FullPath, verbose bool) { |
|
|
|
req, err := http.NewRequest(http.MethodDelete, "", nil) |
|
|
|
|
|
|
|
req.URL = &url.URL{ |
|
|
|
Scheme: "http", |
|
|
|
Host: c.env.option.FilerAddress.ToHttpAddress(), |
|
|
|
Path: string(item.path), |
|
|
|
Path: string(path), |
|
|
|
} |
|
|
|
if verbose { |
|
|
|
fmt.Printf("full HTTP delete request to be sent: %v\n", req) |
|
|
@ -449,10 +450,6 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo |
|
|
|
fmt.Println("delete response Status : ", resp.Status) |
|
|
|
fmt.Println("delete response Headers : ", resp.Header) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { |
|
|
|