@ -6,7 +6,10 @@ import (
"flag"
"flag"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"math"
"math"
"net/http"
"net/url"
"os"
"os"
"path/filepath"
"path/filepath"
"sync"
"sync"
@ -61,7 +64,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
verbose := fsckCommand . Bool ( "v" , false , "verbose mode" )
verbose := fsckCommand . Bool ( "v" , false , "verbose mode" )
findMissingChunksInFiler := fsckCommand . Bool ( "findMissingChunksInFiler" , false , "see \"help volume.fsck\"" )
findMissingChunksInFiler := fsckCommand . Bool ( "findMissingChunksInFiler" , false , "see \"help volume.fsck\"" )
findMissingChunksInFilerPath := fsckCommand . String ( "findMissingChunksInFilerPath" , "/" , "used together with findMissingChunksInFiler" )
findMissingChunksInFilerPath := fsckCommand . String ( "findMissingChunksInFilerPath" , "/" , "used together with findMissingChunksInFiler" )
applyPurging := fsckCommand . Bool ( "reallyDeleteFromVolum e" , false , "<expert only> delete data not referenced by the filer" )
applyPurging := fsckCommand . Bool ( "forcePurg e" , false , "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer" )
if err = fsckCommand . Parse ( args ) ; err != nil {
if err = fsckCommand . Parse ( args ) ; err != nil {
return nil
return nil
}
}
@ -98,7 +101,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
if * findMissingChunksInFiler {
if * findMissingChunksInFiler {
// collect all filer file ids and paths
// collect all filer file ids and paths
if err = c . collectFilerFileIdAndPaths ( volumeIdToVInfo , tempFolder , writer , * findMissingChunksInFilerPath , * verbose , applyPurging ) ; err != nil {
if err = c . collectFilerFileIdAndPaths ( volumeIdToVInfo , tempFolder , writer , * findMissingChunksInFilerPath , * verbose ) ; err != nil {
return fmt . Errorf ( "collectFilerFileIdAndPaths: %v" , err )
return fmt . Errorf ( "collectFilerFileIdAndPaths: %v" , err )
}
}
// for each volume, check filer file ids
// for each volume, check filer file ids
@ -107,10 +110,10 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
}
}
} else {
} else {
// collect all filer file ids
// collect all filer file ids
if err = c . collectFilerFileIds ( tempFolder , volumeIdToVInfo , * verbose , writer ) ; err != nil {
if err = c . collectFilerFileIds ( volumeIdToVInfo , tempFolder , writer , * verbose ) ; err != nil {
return fmt . Errorf ( "failed to collect file ids from filer: %v" , err )
return fmt . Errorf ( "failed to collect file ids from filer: %v" , err )
}
}
// volume file ids subs tract filer file ids
// volume file ids subtract filer file ids
if err = c . findExtraChunksInVolumeServers ( volumeIdToVInfo , tempFolder , writer , * verbose , applyPurging ) ; err != nil {
if err = c . findExtraChunksInVolumeServers ( volumeIdToVInfo , tempFolder , writer , * verbose , applyPurging ) ; err != nil {
return fmt . Errorf ( "findExtraChunksInVolumeServers: %v" , err )
return fmt . Errorf ( "findExtraChunksInVolumeServers: %v" , err )
}
}
@ -119,7 +122,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
return nil
return nil
}
}
func ( c * commandVolumeFsck ) collectFilerFileIdAndPaths ( volumeIdToServer map [ uint32 ] VInfo , tempFolder string , writer io . Writer , filerPath string , verbose bool , applyPurging * bool ) error {
func ( c * commandVolumeFsck ) collectFilerFileIdAndPaths ( volumeIdToServer map [ uint32 ] VInfo , tempFolder string , writer io . Writer , filerPath string , verbose bool ) error {
if verbose {
if verbose {
fmt . Fprintf ( writer , "checking each file from filer ...\n" )
fmt . Fprintf ( writer , "checking each file from filer ...\n" )
@ -185,7 +188,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint
func ( c * commandVolumeFsck ) findFilerChunksMissingInVolumeServers ( volumeIdToVInfo map [ uint32 ] VInfo , tempFolder string , writer io . Writer , verbose bool , applyPurging * bool ) error {
func ( c * commandVolumeFsck ) findFilerChunksMissingInVolumeServers ( volumeIdToVInfo map [ uint32 ] VInfo , tempFolder string , writer io . Writer , verbose bool , applyPurging * bool ) error {
for volumeId , vinfo := range volumeIdToVInfo {
for volumeId , vinfo := range volumeIdToVInfo {
checkErr := c . oneVolumeFileIdsCheckOneVolume ( tempFolder , volumeId , writer , verbose )
checkErr := c . oneVolumeFileIdsCheckOneVolume ( tempFolder , volumeId , writer , verbose , applyPurging )
if checkErr != nil {
if checkErr != nil {
return fmt . Errorf ( "failed to collect file ids from volume %d on %s: %v" , volumeId , vinfo . server , checkErr )
return fmt . Errorf ( "failed to collect file ids from volume %d on %s: %v" , volumeId , vinfo . server , checkErr )
}
}
@ -194,7 +197,9 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf
}
}
func ( c * commandVolumeFsck ) findExtraChunksInVolumeServers ( volumeIdToVInfo map [ uint32 ] VInfo , tempFolder string , writer io . Writer , verbose bool , applyPurging * bool ) error {
func ( c * commandVolumeFsck ) findExtraChunksInVolumeServers ( volumeIdToVInfo map [ uint32 ] VInfo , tempFolder string , writer io . Writer , verbose bool , applyPurging * bool ) error {
var totalInUseCount , totalOrphanChunkCount , totalOrphanDataSize uint64
var totalInUseCount , totalOrphanChunkCount , totalOrphanDataSize uint64
for volumeId , vinfo := range volumeIdToVInfo {
for volumeId , vinfo := range volumeIdToVInfo {
inUseCount , orphanFileIds , orphanDataSize , checkErr := c . oneVolumeFileIdsSubtractFilerFileIds ( tempFolder , volumeId , writer , verbose )
inUseCount , orphanFileIds , orphanDataSize , checkErr := c . oneVolumeFileIdsSubtractFilerFileIds ( tempFolder , volumeId , writer , verbose )
if checkErr != nil {
if checkErr != nil {
@ -210,32 +215,52 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u
}
}
}
}
if * applyPurging && len ( orphanFileIds ) > 0 {
if len ( orphanFileIds ) > 0 {
if * applyPurging {
if verbose {
fmt . Fprintf ( writer , "purging process for volume %d" , volumeId )
}
if vinfo . isEcVolume {
if vinfo . isEcVolume {
fmt . Fprintf ( writer , "Skip purging for Erasure Coded volume %d.\n" , volumeId )
fmt . Fprintf ( writer , "s kip purging for Erasure Coded volume %d.\n" , volumeId )
continue
continue
}
}
needleVID := needle . VolumeId ( volumeId )
if vinfo . isReadOnly {
if vinfo . isReadOnly {
fmt . Fprintf ( writer , "Skip purging for read only volume %d.\n" , volumeId )
continue
err := markVolumeWritable ( c . env . option . GrpcDialOption , needleVID , vinfo . server , true )
if err != nil {
return fmt . Errorf ( "mark volume %d read/write: %v" , volumeId , err )
}
}
if inUseCount == 0 {
if err := deleteVolume ( c . env . option . GrpcDialOption , needle . VolumeId ( volumeId ) , vinfo . server ) ; err != nil {
return fmt . Errorf ( "delete volume %d: %v" , volumeId , err )
fmt . Fprintf ( writer , "temporarily marked %d on server %v writable for forced purge\n" , volumeId , vinfo . server )
defer markVolumeWritable ( c . env . option . GrpcDialOption , needleVID , vinfo . server , false )
}
fmt . Fprintf ( writer , "marked %d on server %v writable for forced purge\n" , volumeId , vinfo . server )
// FIXME Does not delete all replicas of this volumeId on all volume servers. Using fileId based deletion in all cases for now.
// if inUseCount < 1 {
// if verbose {
// fmt.Fprintf(writer, "removing empty (all content orphaned) volume %d\n", volumeId)
// }
// if err := deleteVolume(c.env.option.GrpcDialOption, needleVID, vinfo.server); err != nil {
// return fmt.Errorf("removing empty volume %d: %v", volumeId, err)
// }
// } else {
if verbose {
fmt . Fprintf ( writer , "purging files from volume %d\n" , volumeId )
}
}
} else {
if err := c . purgeFileIdsForOneVolume ( volumeId , orphanFileIds , writer ) ; err != nil {
if err := c . purgeFileIdsForOneVolume ( volumeId , orphanFileIds , writer ) ; err != nil {
return fmt . Errorf ( "purge for volume %d: %v" , volumeId , err )
return fmt . Errorf ( "purging volume %d: %v" , volumeId , err )
}
}
}
}
}
}
}
}
if totalOrphanChunkCount == 0 {
fmt . Fprintf ( writer , "no orphan data\n" )
return nil
}
if ! * applyPurging {
if ! * applyPurging {
pct := float64 ( totalOrphanChunkCount * 100 ) / ( float64 ( totalOrphanChunkCount + totalInUseCount ) )
pct := float64 ( totalOrphanChunkCount * 100 ) / ( float64 ( totalOrphanChunkCount + totalInUseCount ) )
fmt . Fprintf ( writer , "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n" ,
fmt . Fprintf ( writer , "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n" ,
@ -243,6 +268,12 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[u
fmt . Fprintf ( writer , "This could be normal if multiple filers or no filers are used.\n" )
fmt . Fprintf ( writer , "This could be normal if multiple filers or no filers are used.\n" )
}
}
if totalOrphanChunkCount == 0 {
fmt . Fprintf ( writer , "no orphan data\n" )
//return nil
}
return nil
return nil
}
}
@ -283,7 +314,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId
}
}
func ( c * commandVolumeFsck ) collectFilerFileIds ( tempFolder string , volumeIdToServer map [ uint32 ] VInfo , verbose bool , writer io . Writer ) error {
func ( c * commandVolumeFsck ) collectFilerFileIds ( volumeIdToServer map [ uint32 ] VInfo , tempFolder string , writer io . Writer , verbose bool ) error {
if verbose {
if verbose {
fmt . Fprintf ( writer , "collecting file ids from filer ...\n" )
fmt . Fprintf ( writer , "collecting file ids from filer ...\n" )
@ -333,10 +364,10 @@ func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToSer
} )
} )
}
}
func ( c * commandVolumeFsck ) oneVolumeFileIdsCheckOneVolume ( tempFolder string , volumeId uint32 , writer io . Writer , verbose bool ) ( err error ) {
func ( c * commandVolumeFsck ) oneVolumeFileIdsCheckOneVolume ( tempFolder string , volumeId uint32 , writer io . Writer , verbose bool , applyPurging * bool ) ( err error ) {
if verbose {
if verbose {
fmt . Fprintf ( writer , "find missing file chuns in volume %d ...\n" , volumeId )
fmt . Fprintf ( writer , "find missing file chunk s in volume %d ...\n" , volumeId )
}
}
db := needle_map . NewMemDb ( )
db := needle_map . NewMemDb ( )
@ -366,12 +397,8 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo
for {
for {
readSize , err = io . ReadFull ( br , buffer )
readSize , err = io . ReadFull ( br , buffer )
if err != nil || readSize != 16 {
if err != nil || readSize != 16 {
if err == io . EOF {
return nil
} else {
break
break
}
}
}
item . fileKey = util . BytesToUint64 ( buffer [ : 8 ] )
item . fileKey = util . BytesToUint64 ( buffer [ : 8 ] )
item . cookie = util . BytesToUint32 ( buffer [ 8 : 12 ] )
item . cookie = util . BytesToUint32 ( buffer [ 8 : 12 ] )
@ -386,14 +413,46 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, vo
}
}
item . path = util . FullPath ( string ( pathBytes ) )
item . path = util . FullPath ( string ( pathBytes ) )
if _ , found := db . Get ( types . NeedleId ( item . fileKey ) ) ; ! found {
fmt . Fprintf ( writer , "%d,%x%08x in %s %d not found\n" , volumeId , item . fileKey , item . cookie , item . path , pathSize )
needleId := types . NeedleId ( item . fileKey )
if _ , found := db . Get ( needleId ) ; ! found {
fmt . Fprintf ( writer , "%s\n" , item . path )
if * applyPurging {
req , err := http . NewRequest ( http . MethodDelete , "" , nil )
// defining the URL this way automatically escapes complex path names
req . URL = & url . URL {
Scheme : "http" ,
Host : c . env . option . FilerAddress . ToHttpAddress ( ) ,
Path : string ( item . path ) ,
}
if verbose {
fmt . Printf ( "full HTTP delete request to be sent: %v\n" , req )
}
if err != nil {
fmt . Errorf ( "HTTP delete request error: %v\n" , err )
}
}
client := & http . Client { }
resp , err := client . Do ( req )
if err != nil {
fmt . Errorf ( "DELETE fetch error: %v\n" , err )
}
}
defer resp . Body . Close ( )
return
_ , err = ioutil . ReadAll ( resp . Body )
if err != nil {
fmt . Errorf ( "DELETE response error: %v\n" , err )
}
if verbose {
fmt . Println ( "delete response Status : " , resp . Status )
fmt . Println ( "delete response Headers : " , resp . Header )
}
}
}
}
return nil
}
}
func ( c * commandVolumeFsck ) oneVolumeFileIdsSubtractFilerFileIds ( tempFolder string , volumeId uint32 , writer io . Writer , verbose bool ) ( inUseCount uint64 , orphanFileIds [ ] string , orphanDataSize uint64 , err error ) {
func ( c * commandVolumeFsck ) oneVolumeFileIdsSubtractFilerFileIds ( tempFolder string , volumeId uint32 , writer io . Writer , verbose bool ) ( inUseCount uint64 , orphanFileIds [ ] string , orphanDataSize uint64 , err error ) {