You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

546 lines
16 KiB

4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
  1. package shell
  2. import (
  3. "bufio"
  4. "context"
  5. "flag"
  6. "fmt"
  7. "io"
  8. "math"
  9. "os"
  10. "path/filepath"
  11. "sync"
  12. "github.com/chrislusf/seaweedfs/weed/filer"
  13. "github.com/chrislusf/seaweedfs/weed/operation"
  14. "github.com/chrislusf/seaweedfs/weed/pb"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  17. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  18. "github.com/chrislusf/seaweedfs/weed/storage/needle"
  19. "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
  20. "github.com/chrislusf/seaweedfs/weed/storage/types"
  21. "github.com/chrislusf/seaweedfs/weed/util"
  22. )
  23. func init() {
  24. Commands = append(Commands, &commandVolumeFsck{})
  25. }
  26. type commandVolumeFsck struct {
  27. env *CommandEnv
  28. }
  29. func (c *commandVolumeFsck) Name() string {
  30. return "volume.fsck"
  31. }
  32. func (c *commandVolumeFsck) Help() string {
  33. return `check all volumes to find entries not used by the filer
  34. Important assumption!!!
  35. the system is all used by one filer.
  36. This command works this way:
  37. 1. collect all file ids from all volumes, as set A
  38. 2. collect all file ids from the filer, as set B
  39. 3. find out the set A subtract B
  40. If -findMissingChunksInFiler is enabled, this works
  41. in a reverse way:
  42. 1. collect all file ids from all volumes, as set A
  43. 2. collect all file ids from the filer, as set B
  44. 3. find out the set B subtract A
  45. `
  46. }
  47. func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  48. fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  49. verbose := fsckCommand.Bool("v", false, "verbose mode")
  50. findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"")
  51. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler")
  52. applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only> delete data not referenced by the filer")
  53. if err = fsckCommand.Parse(args); err != nil {
  54. return nil
  55. }
  56. if err = commandEnv.confirmIsLocked(args); err != nil {
  57. return
  58. }
  59. c.env = commandEnv
  60. // create a temp folder
  61. tempFolder, err := os.MkdirTemp("", "sw_fsck")
  62. if err != nil {
  63. return fmt.Errorf("failed to create temp folder: %v", err)
  64. }
  65. if *verbose {
  66. fmt.Fprintf(writer, "working directory: %s\n", tempFolder)
  67. }
  68. defer os.RemoveAll(tempFolder)
  69. // collect all volume id locations
  70. volumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer)
  71. if err != nil {
  72. return fmt.Errorf("failed to collect all volume locations: %v", err)
  73. }
  74. // collect each volume file ids
  75. for volumeId, vinfo := range volumeIdToVInfo {
  76. err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose, writer)
  77. if err != nil {
  78. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err)
  79. }
  80. }
  81. if *findMissingChunksInFiler {
  82. // collect all filer file ids and paths
  83. if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, applyPurging); err != nil {
  84. return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
  85. }
  86. // for each volume, check filer file ids
  87. if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
  88. return fmt.Errorf("findFilerChunksMissingInVolumeServers: %v", err)
  89. }
  90. } else {
  91. // collect all filer file ids
  92. if err = c.collectFilerFileIds(tempFolder, volumeIdToVInfo, *verbose, writer); err != nil {
  93. return fmt.Errorf("failed to collect file ids from filer: %v", err)
  94. }
  95. // volume file ids substract filer file ids
  96. if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
  97. return fmt.Errorf("findExtraChunksInVolumeServers: %v", err)
  98. }
  99. }
  100. return nil
  101. }
  102. func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging *bool) error {
  103. if verbose {
  104. fmt.Fprintf(writer, "checking each file from filer ...\n")
  105. }
  106. files := make(map[uint32]*os.File)
  107. for vid := range volumeIdToServer {
  108. dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  109. if openErr != nil {
  110. return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
  111. }
  112. files[vid] = dst
  113. }
  114. defer func() {
  115. for _, f := range files {
  116. f.Close()
  117. }
  118. }()
  119. type Item struct {
  120. vid uint32
  121. fileKey uint64
  122. cookie uint32
  123. path util.FullPath
  124. }
  125. return doTraverseBfsAndSaving(c.env, nil, filerPath, false, func(outputChan chan interface{}) {
  126. buffer := make([]byte, 16)
  127. for item := range outputChan {
  128. i := item.(*Item)
  129. if f, ok := files[i.vid]; ok {
  130. util.Uint64toBytes(buffer, i.fileKey)
  131. util.Uint32toBytes(buffer[8:], i.cookie)
  132. util.Uint32toBytes(buffer[12:], uint32(len(i.path)))
  133. f.Write(buffer)
  134. f.Write([]byte(i.path))
  135. // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path)
  136. } else {
  137. fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
  138. }
  139. }
  140. }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
  141. if verbose && entry.Entry.IsDirectory {
  142. fmt.Fprintf(writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))
  143. }
  144. dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64)
  145. if resolveErr != nil {
  146. return nil
  147. }
  148. dChunks = append(dChunks, mChunks...)
  149. for _, chunk := range dChunks {
  150. outputChan <- &Item{
  151. vid: chunk.Fid.VolumeId,
  152. fileKey: chunk.Fid.FileKey,
  153. cookie: chunk.Fid.Cookie,
  154. path: util.NewFullPath(entry.Dir, entry.Entry.Name),
  155. }
  156. }
  157. return nil
  158. })
  159. }
  160. func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
  161. for volumeId, vinfo := range volumeIdToVInfo {
  162. checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose)
  163. if checkErr != nil {
  164. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
  165. }
  166. }
  167. return nil
  168. }
  169. func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
  170. var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64
  171. for volumeId, vinfo := range volumeIdToVInfo {
  172. inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose)
  173. if checkErr != nil {
  174. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
  175. }
  176. totalInUseCount += inUseCount
  177. totalOrphanChunkCount += uint64(len(orphanFileIds))
  178. totalOrphanDataSize += orphanDataSize
  179. if verbose {
  180. for _, fid := range orphanFileIds {
  181. fmt.Fprintf(writer, "%s\n", fid)
  182. }
  183. }
  184. if *applyPurging && len(orphanFileIds) > 0 {
  185. if vinfo.isEcVolume {
  186. fmt.Fprintf(writer, "Skip purging for Erasure Coded volume %d.\n", volumeId)
  187. continue
  188. }
  189. if inUseCount == 0 {
  190. if err := deleteVolume(c.env.option.GrpcDialOption, needle.VolumeId(volumeId), vinfo.server); err != nil {
  191. return fmt.Errorf("delete volume %d: %v", volumeId, err)
  192. }
  193. } else {
  194. if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil {
  195. return fmt.Errorf("purge for volume %d: %v", volumeId, err)
  196. }
  197. }
  198. }
  199. }
  200. if totalOrphanChunkCount == 0 {
  201. fmt.Fprintf(writer, "no orphan data\n")
  202. return nil
  203. }
  204. if !*applyPurging {
  205. pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
  206. fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
  207. totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)
  208. fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n")
  209. }
  210. return nil
  211. }
  212. func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error {
  213. if verbose {
  214. fmt.Fprintf(writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server)
  215. }
  216. return operation.WithVolumeServerClient(vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
  217. ext := ".idx"
  218. if vinfo.isEcVolume {
  219. ext = ".ecx"
  220. }
  221. copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
  222. VolumeId: volumeId,
  223. Ext: ext,
  224. CompactionRevision: math.MaxUint32,
  225. StopOffset: math.MaxInt64,
  226. Collection: vinfo.collection,
  227. IsEcVolume: vinfo.isEcVolume,
  228. IgnoreSourceFileNotFound: false,
  229. })
  230. if err != nil {
  231. return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
  232. }
  233. err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
  234. if err != nil {
  235. return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
  236. }
  237. return nil
  238. })
  239. }
  240. func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo, verbose bool, writer io.Writer) error {
  241. if verbose {
  242. fmt.Fprintf(writer, "collecting file ids from filer ...\n")
  243. }
  244. files := make(map[uint32]*os.File)
  245. for vid := range volumeIdToServer {
  246. dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  247. if openErr != nil {
  248. return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
  249. }
  250. files[vid] = dst
  251. }
  252. defer func() {
  253. for _, f := range files {
  254. f.Close()
  255. }
  256. }()
  257. type Item struct {
  258. vid uint32
  259. fileKey uint64
  260. }
  261. return doTraverseBfsAndSaving(c.env, nil, "/", false, func(outputChan chan interface{}) {
  262. buffer := make([]byte, 8)
  263. for item := range outputChan {
  264. i := item.(*Item)
  265. util.Uint64toBytes(buffer, i.fileKey)
  266. files[i.vid].Write(buffer)
  267. }
  268. }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
  269. dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks, 0, math.MaxInt64)
  270. if resolveErr != nil {
  271. if verbose {
  272. fmt.Fprintf(writer, "resolving manifest chunks in %s: %v\n", util.NewFullPath(entry.Dir, entry.Entry.Name), resolveErr)
  273. }
  274. return nil
  275. }
  276. dChunks = append(dChunks, mChunks...)
  277. for _, chunk := range dChunks {
  278. outputChan <- &Item{
  279. vid: chunk.Fid.VolumeId,
  280. fileKey: chunk.Fid.FileKey,
  281. }
  282. }
  283. return nil
  284. })
  285. }
  286. func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (err error) {
  287. if verbose {
  288. fmt.Fprintf(writer, "find missing file chuns in volume %d ...\n", volumeId)
  289. }
  290. db := needle_map.NewMemDb()
  291. defer db.Close()
  292. if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
  293. return
  294. }
  295. file := getFilerFileIdFile(tempFolder, volumeId)
  296. fp, err := os.Open(file)
  297. if err != nil {
  298. return
  299. }
  300. defer fp.Close()
  301. type Item struct {
  302. fileKey uint64
  303. cookie uint32
  304. path util.FullPath
  305. }
  306. br := bufio.NewReader(fp)
  307. buffer := make([]byte, 16)
  308. item := &Item{}
  309. var readSize int
  310. for {
  311. readSize, err = io.ReadFull(br, buffer)
  312. if err != nil || readSize != 16 {
  313. if err == io.EOF {
  314. return nil
  315. } else {
  316. break
  317. }
  318. }
  319. item.fileKey = util.BytesToUint64(buffer[:8])
  320. item.cookie = util.BytesToUint32(buffer[8:12])
  321. pathSize := util.BytesToUint32(buffer[12:16])
  322. pathBytes := make([]byte, int(pathSize))
  323. n, err := io.ReadFull(br, pathBytes)
  324. if err != nil {
  325. fmt.Fprintf(writer, "%d,%x%08x in unexpected error: %v\n", volumeId, item.fileKey, item.cookie, err)
  326. }
  327. if n != int(pathSize) {
  328. fmt.Fprintf(writer, "%d,%x%08x %d unexpected file name size %d\n", volumeId, item.fileKey, item.cookie, pathSize, n)
  329. }
  330. item.path = util.FullPath(string(pathBytes))
  331. if _, found := db.Get(types.NeedleId(item.fileKey)); !found {
  332. fmt.Fprintf(writer, "%d,%x%08x in %s %d not found\n", volumeId, item.fileKey, item.cookie, item.path, pathSize)
  333. }
  334. }
  335. return
  336. }
  337. func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) {
  338. db := needle_map.NewMemDb()
  339. defer db.Close()
  340. if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
  341. return
  342. }
  343. filerFileIdsData, err := os.ReadFile(getFilerFileIdFile(tempFolder, volumeId))
  344. if err != nil {
  345. return
  346. }
  347. dataLen := len(filerFileIdsData)
  348. if dataLen%8 != 0 {
  349. return 0, nil, 0, fmt.Errorf("filer data is corrupted")
  350. }
  351. for i := 0; i < len(filerFileIdsData); i += 8 {
  352. fileKey := util.BytesToUint64(filerFileIdsData[i : i+8])
  353. db.Delete(types.NeedleId(fileKey))
  354. inUseCount++
  355. }
  356. var orphanFileCount uint64
  357. db.AscendingVisit(func(n needle_map.NeedleValue) error {
  358. // fmt.Printf("%d,%x\n", volumeId, n.Key)
  359. orphanFileIds = append(orphanFileIds, fmt.Sprintf("%d,%s00000000", volumeId, n.Key.String()))
  360. orphanFileCount++
  361. orphanDataSize += uint64(n.Size)
  362. return nil
  363. })
  364. if orphanFileCount > 0 {
  365. pct := float64(orphanFileCount*100) / (float64(orphanFileCount + inUseCount))
  366. fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
  367. volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize)
  368. }
  369. return
  370. }
  371. type VInfo struct {
  372. server pb.ServerAddress
  373. collection string
  374. isEcVolume bool
  375. }
  376. func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[uint32]VInfo, err error) {
  377. if verbose {
  378. fmt.Fprintf(writer, "collecting volume id and locations from master ...\n")
  379. }
  380. volumeIdToServer = make(map[uint32]VInfo)
  381. // collect topology information
  382. topologyInfo, _, err := collectTopologyInfo(commandEnv)
  383. if err != nil {
  384. return
  385. }
  386. eachDataNode(topologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) {
  387. for _, diskInfo := range t.DiskInfos {
  388. for _, vi := range diskInfo.VolumeInfos {
  389. volumeIdToServer[vi.Id] = VInfo{
  390. server: pb.NewServerAddressFromDataNode(t),
  391. collection: vi.Collection,
  392. isEcVolume: false,
  393. }
  394. }
  395. for _, ecShardInfo := range diskInfo.EcShardInfos {
  396. volumeIdToServer[ecShardInfo.Id] = VInfo{
  397. server: pb.NewServerAddressFromDataNode(t),
  398. collection: ecShardInfo.Collection,
  399. isEcVolume: true,
  400. }
  401. }
  402. }
  403. })
  404. if verbose {
  405. fmt.Fprintf(writer, "collected %d volumes and locations.\n", len(volumeIdToServer))
  406. }
  407. return
  408. }
  409. func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds []string, writer io.Writer) (err error) {
  410. fmt.Fprintf(writer, "purging orphan data for volume %d...\n", volumeId)
  411. locations, found := c.env.MasterClient.GetLocations(volumeId)
  412. if !found {
  413. return fmt.Errorf("failed to find volume %d locations", volumeId)
  414. }
  415. resultChan := make(chan []*volume_server_pb.DeleteResult, len(locations))
  416. var wg sync.WaitGroup
  417. for _, location := range locations {
  418. wg.Add(1)
  419. go func(server pb.ServerAddress, fidList []string) {
  420. defer wg.Done()
  421. if deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil {
  422. err = deleteErr
  423. } else if deleteResults != nil {
  424. resultChan <- deleteResults
  425. }
  426. }(location.ServerAddress(), fileIds)
  427. }
  428. wg.Wait()
  429. close(resultChan)
  430. for results := range resultChan {
  431. for _, result := range results {
  432. if result.Error != "" {
  433. fmt.Fprintf(writer, "purge error: %s\n", result.Error)
  434. }
  435. }
  436. }
  437. return
  438. }
  439. func getVolumeFileIdFile(tempFolder string, vid uint32) string {
  440. return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid))
  441. }
  442. func getFilerFileIdFile(tempFolder string, vid uint32) string {
  443. return filepath.Join(tempFolder, fmt.Sprintf("%d.fid", vid))
  444. }
  445. func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
  446. flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
  447. dst, err := os.OpenFile(fileName, flags, 0644)
  448. if err != nil {
  449. return nil
  450. }
  451. defer dst.Close()
  452. for {
  453. resp, receiveErr := client.Recv()
  454. if receiveErr == io.EOF {
  455. break
  456. }
  457. if receiveErr != nil {
  458. return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
  459. }
  460. dst.Write(resp.FileContent)
  461. }
  462. return nil
  463. }