You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

536 lines
16 KiB

4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
  1. package shell
  2. import (
  3. "bufio"
  4. "context"
  5. "flag"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "math"
  10. "os"
  11. "path/filepath"
  12. "sync"
  13. "github.com/chrislusf/seaweedfs/weed/filer"
  14. "github.com/chrislusf/seaweedfs/weed/operation"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  17. "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
  18. "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
  19. "github.com/chrislusf/seaweedfs/weed/storage/types"
  20. "github.com/chrislusf/seaweedfs/weed/util"
  21. )
  22. func init() {
  23. Commands = append(Commands, &commandVolumeFsck{})
  24. }
  25. type commandVolumeFsck struct {
  26. env *CommandEnv
  27. }
  28. func (c *commandVolumeFsck) Name() string {
  29. return "volume.fsck"
  30. }
  31. func (c *commandVolumeFsck) Help() string {
  32. return `check all volumes to find entries not used by the filer
  33. Important assumption!!!
  34. the system is all used by one filer.
  35. This command works this way:
  36. 1. collect all file ids from all volumes, as set A
  37. 2. collect all file ids from the filer, as set B
  38. 3. find out the set A subtract B
  39. If -findMissingChunksInFiler is enabled, this works
  40. in a reverse way:
  41. 1. collect all file ids from all volumes, as set A
  42. 2. collect all file ids from the filer, as set B
  43. 3. find out the set B subtract A
  44. `
  45. }
  46. func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  47. if err = commandEnv.confirmIsLocked(); err != nil {
  48. return
  49. }
  50. fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  51. verbose := fsckCommand.Bool("v", false, "verbose mode")
  52. findMissingChunksInFiler := fsckCommand.Bool("findMissingChunksInFiler", false, "see \"help volume.fsck\"")
  53. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler")
  54. applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only> delete data not referenced by the filer")
  55. if err = fsckCommand.Parse(args); err != nil {
  56. return nil
  57. }
  58. c.env = commandEnv
  59. // create a temp folder
  60. tempFolder, err := ioutil.TempDir("", "sw_fsck")
  61. if err != nil {
  62. return fmt.Errorf("failed to create temp folder: %v", err)
  63. }
  64. if *verbose {
  65. fmt.Fprintf(writer, "working directory: %s\n", tempFolder)
  66. }
  67. defer os.RemoveAll(tempFolder)
  68. // collect all volume id locations
  69. volumeIdToVInfo, err := c.collectVolumeIds(commandEnv, *verbose, writer)
  70. if err != nil {
  71. return fmt.Errorf("failed to collect all volume locations: %v", err)
  72. }
  73. // collect each volume file ids
  74. for volumeId, vinfo := range volumeIdToVInfo {
  75. err = c.collectOneVolumeFileIds(tempFolder, volumeId, vinfo, *verbose, writer)
  76. if err != nil {
  77. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err)
  78. }
  79. }
  80. if *findMissingChunksInFiler {
  81. // collect all filer file ids and paths
  82. if err = c.collectFilerFileIdAndPaths(volumeIdToVInfo, tempFolder, writer, *findMissingChunksInFilerPath, *verbose, applyPurging); err != nil {
  83. return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
  84. }
  85. // for each volume, check filer file ids
  86. if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
  87. return fmt.Errorf("findExtraChunksInVolumeServers: %v", err)
  88. }
  89. } else {
  90. // collect all filer file ids
  91. if err = c.collectFilerFileIds(tempFolder, volumeIdToVInfo, *verbose, writer); err != nil {
  92. return fmt.Errorf("failed to collect file ids from filer: %v", err)
  93. }
  94. // volume file ids substract filer file ids
  95. if err = c.findExtraChunksInVolumeServers(volumeIdToVInfo, tempFolder, writer, *verbose, applyPurging); err != nil {
  96. return fmt.Errorf("findExtraChunksInVolumeServers: %v", err)
  97. }
  98. }
  99. return nil
  100. }
  101. func (c *commandVolumeFsck) collectFilerFileIdAndPaths(volumeIdToServer map[uint32]VInfo, tempFolder string, writer io.Writer, filerPath string, verbose bool, applyPurging *bool) error {
  102. if verbose {
  103. fmt.Fprintf(writer, "checking each file from filer ...\n")
  104. }
  105. files := make(map[uint32]*os.File)
  106. for vid := range volumeIdToServer {
  107. dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  108. if openErr != nil {
  109. return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
  110. }
  111. files[vid] = dst
  112. }
  113. defer func() {
  114. for _, f := range files {
  115. f.Close()
  116. }
  117. }()
  118. type Item struct {
  119. vid uint32
  120. fileKey uint64
  121. cookie uint32
  122. path util.FullPath
  123. }
  124. return doTraverseBfsAndSaving(c.env, nil, filerPath, false, func(outputChan chan interface{}) {
  125. buffer := make([]byte, 16)
  126. for item := range outputChan {
  127. i := item.(*Item)
  128. if f, ok := files[i.vid]; ok {
  129. util.Uint64toBytes(buffer, i.fileKey)
  130. util.Uint32toBytes(buffer[8:], i.cookie)
  131. util.Uint32toBytes(buffer[12:], uint32(len(i.path)))
  132. f.Write(buffer)
  133. f.Write([]byte(i.path))
  134. // fmt.Fprintf(writer, "%d,%x%08x %d %s\n", i.vid, i.fileKey, i.cookie, len(i.path), i.path)
  135. } else {
  136. fmt.Fprintf(writer, "%d,%x%08x %s volume not found\n", i.vid, i.fileKey, i.cookie, i.path)
  137. }
  138. }
  139. }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
  140. if verbose && entry.Entry.IsDirectory {
  141. fmt.Fprintf(writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))
  142. }
  143. dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks)
  144. if resolveErr != nil {
  145. return nil
  146. }
  147. dChunks = append(dChunks, mChunks...)
  148. for _, chunk := range dChunks {
  149. outputChan <- &Item{
  150. vid: chunk.Fid.VolumeId,
  151. fileKey: chunk.Fid.FileKey,
  152. cookie: chunk.Fid.Cookie,
  153. path: util.NewFullPath(entry.Dir, entry.Entry.Name),
  154. }
  155. }
  156. return nil
  157. })
  158. return nil
  159. }
  160. func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
  161. for volumeId, vinfo := range volumeIdToVInfo {
  162. checkErr := c.oneVolumeFileIdsCheckOneVolume(tempFolder, volumeId, writer, verbose)
  163. if checkErr != nil {
  164. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
  165. }
  166. }
  167. return nil
  168. }
  169. func (c *commandVolumeFsck) findExtraChunksInVolumeServers(volumeIdToVInfo map[uint32]VInfo, tempFolder string, writer io.Writer, verbose bool, applyPurging *bool) error {
  170. var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64
  171. for volumeId, vinfo := range volumeIdToVInfo {
  172. inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(tempFolder, volumeId, writer, verbose)
  173. if checkErr != nil {
  174. return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr)
  175. }
  176. totalInUseCount += inUseCount
  177. totalOrphanChunkCount += uint64(len(orphanFileIds))
  178. totalOrphanDataSize += orphanDataSize
  179. if verbose {
  180. for _, fid := range orphanFileIds {
  181. fmt.Fprintf(writer, "%sxxxxxxxx\n", fid)
  182. }
  183. }
  184. if *applyPurging && len(orphanFileIds) > 0 {
  185. if vinfo.isEcVolume {
  186. fmt.Fprintf(writer, "Skip purging for Erasure Coded volumes.\n")
  187. }
  188. if err := c.purgeFileIdsForOneVolume(volumeId, orphanFileIds, writer); err != nil {
  189. return fmt.Errorf("purge for volume %d: %v\n", volumeId, err)
  190. }
  191. }
  192. }
  193. if totalOrphanChunkCount == 0 {
  194. fmt.Fprintf(writer, "no orphan data\n")
  195. return nil
  196. }
  197. if !*applyPurging {
  198. pct := float64(totalOrphanChunkCount*100) / (float64(totalOrphanChunkCount + totalInUseCount))
  199. fmt.Fprintf(writer, "\nTotal\t\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
  200. totalOrphanChunkCount+totalInUseCount, totalOrphanChunkCount, pct, totalOrphanDataSize)
  201. fmt.Fprintf(writer, "This could be normal if multiple filers or no filers are used.\n")
  202. }
  203. return nil
  204. }
  205. func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, volumeId uint32, vinfo VInfo, verbose bool, writer io.Writer) error {
  206. if verbose {
  207. fmt.Fprintf(writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server)
  208. }
  209. return operation.WithVolumeServerClient(vinfo.server, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
  210. ext := ".idx"
  211. if vinfo.isEcVolume {
  212. ext = ".ecx"
  213. }
  214. copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
  215. VolumeId: volumeId,
  216. Ext: ext,
  217. CompactionRevision: math.MaxUint32,
  218. StopOffset: math.MaxInt64,
  219. Collection: vinfo.collection,
  220. IsEcVolume: vinfo.isEcVolume,
  221. IgnoreSourceFileNotFound: false,
  222. })
  223. if err != nil {
  224. return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
  225. }
  226. err = writeToFile(copyFileClient, getVolumeFileIdFile(tempFolder, volumeId))
  227. if err != nil {
  228. return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err)
  229. }
  230. return nil
  231. })
  232. }
  233. func (c *commandVolumeFsck) collectFilerFileIds(tempFolder string, volumeIdToServer map[uint32]VInfo, verbose bool, writer io.Writer) error {
  234. if verbose {
  235. fmt.Fprintf(writer, "collecting file ids from filer ...\n")
  236. }
  237. files := make(map[uint32]*os.File)
  238. for vid := range volumeIdToServer {
  239. dst, openErr := os.OpenFile(getFilerFileIdFile(tempFolder, vid), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  240. if openErr != nil {
  241. return fmt.Errorf("failed to create file %s: %v", getFilerFileIdFile(tempFolder, vid), openErr)
  242. }
  243. files[vid] = dst
  244. }
  245. defer func() {
  246. for _, f := range files {
  247. f.Close()
  248. }
  249. }()
  250. type Item struct {
  251. vid uint32
  252. fileKey uint64
  253. }
  254. return doTraverseBfsAndSaving(c.env, nil, "/", false, func(outputChan chan interface{}) {
  255. buffer := make([]byte, 8)
  256. for item := range outputChan {
  257. i := item.(*Item)
  258. util.Uint64toBytes(buffer, i.fileKey)
  259. files[i.vid].Write(buffer)
  260. }
  261. }, func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
  262. dChunks, mChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.Chunks)
  263. if resolveErr != nil {
  264. return nil
  265. }
  266. dChunks = append(dChunks, mChunks...)
  267. for _, chunk := range dChunks {
  268. outputChan <- &Item{
  269. vid: chunk.Fid.VolumeId,
  270. fileKey: chunk.Fid.FileKey,
  271. }
  272. }
  273. return nil
  274. })
  275. }
  276. func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (err error) {
  277. if verbose {
  278. fmt.Fprintf(writer, "find missing file chuns in volume %d ...\n", volumeId)
  279. }
  280. db := needle_map.NewMemDb()
  281. defer db.Close()
  282. if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
  283. return
  284. }
  285. file := getFilerFileIdFile(tempFolder, volumeId)
  286. fp, err := os.Open(file)
  287. if err != nil {
  288. return
  289. }
  290. defer fp.Close()
  291. type Item struct {
  292. fileKey uint64
  293. cookie uint32
  294. path util.FullPath
  295. }
  296. br := bufio.NewReader(fp)
  297. buffer := make([]byte, 16)
  298. item := &Item{}
  299. var readSize int
  300. for {
  301. readSize, err = io.ReadFull(br, buffer)
  302. if err != nil || readSize != 16 {
  303. if err == io.EOF {
  304. return nil
  305. } else {
  306. break
  307. }
  308. }
  309. item.fileKey = util.BytesToUint64(buffer[:8])
  310. item.cookie = util.BytesToUint32(buffer[8:12])
  311. pathSize := util.BytesToUint32(buffer[12:16])
  312. pathBytes := make([]byte, int(pathSize))
  313. n, err := io.ReadFull(br, pathBytes)
  314. if err != nil {
  315. fmt.Fprintf(writer, "%d,%x%08x in unexpected error: %v\n", volumeId, item.fileKey, item.cookie, err)
  316. }
  317. if n != int(pathSize) {
  318. fmt.Fprintf(writer, "%d,%x%08x %d unexpected file name size %d\n", volumeId, item.fileKey, item.cookie, pathSize, n)
  319. }
  320. item.path = util.FullPath(string(pathBytes))
  321. if _, found := db.Get(types.NeedleId(item.fileKey)); !found {
  322. fmt.Fprintf(writer, "%d,%x%08x in %s %d not found\n", volumeId, item.fileKey, item.cookie, item.path, pathSize)
  323. }
  324. }
  325. return
  326. }
  327. func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(tempFolder string, volumeId uint32, writer io.Writer, verbose bool) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) {
  328. db := needle_map.NewMemDb()
  329. defer db.Close()
  330. if err = db.LoadFromIdx(getVolumeFileIdFile(tempFolder, volumeId)); err != nil {
  331. return
  332. }
  333. filerFileIdsData, err := ioutil.ReadFile(getFilerFileIdFile(tempFolder, volumeId))
  334. if err != nil {
  335. return
  336. }
  337. dataLen := len(filerFileIdsData)
  338. if dataLen%8 != 0 {
  339. return 0, nil, 0, fmt.Errorf("filer data is corrupted")
  340. }
  341. for i := 0; i < len(filerFileIdsData); i += 8 {
  342. fileKey := util.BytesToUint64(filerFileIdsData[i : i+8])
  343. db.Delete(types.NeedleId(fileKey))
  344. inUseCount++
  345. }
  346. var orphanFileCount uint64
  347. db.AscendingVisit(func(n needle_map.NeedleValue) error {
  348. // fmt.Printf("%d,%x\n", volumeId, n.Key)
  349. orphanFileIds = append(orphanFileIds, fmt.Sprintf("%d,%s", volumeId, n.Key.String()))
  350. orphanFileCount++
  351. orphanDataSize += uint64(n.Size)
  352. return nil
  353. })
  354. if orphanFileCount > 0 {
  355. pct := float64(orphanFileCount*100) / (float64(orphanFileCount + inUseCount))
  356. fmt.Fprintf(writer, "volume:%d\tentries:%d\torphan:%d\t%.2f%%\t%dB\n",
  357. volumeId, orphanFileCount+inUseCount, orphanFileCount, pct, orphanDataSize)
  358. }
  359. return
  360. }
  361. type VInfo struct {
  362. server string
  363. collection string
  364. isEcVolume bool
  365. }
  366. func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose bool, writer io.Writer) (volumeIdToServer map[uint32]VInfo, err error) {
  367. if verbose {
  368. fmt.Fprintf(writer, "collecting volume id and locations from master ...\n")
  369. }
  370. volumeIdToServer = make(map[uint32]VInfo)
  371. // collect topology information
  372. topologyInfo, _, err := collectTopologyInfo(commandEnv)
  373. if err != nil {
  374. return
  375. }
  376. eachDataNode(topologyInfo, func(dc string, rack RackId, t *master_pb.DataNodeInfo) {
  377. for _, diskInfo := range t.DiskInfos {
  378. for _, vi := range diskInfo.VolumeInfos {
  379. volumeIdToServer[vi.Id] = VInfo{
  380. server: t.Id,
  381. collection: vi.Collection,
  382. isEcVolume: false,
  383. }
  384. }
  385. for _, ecShardInfo := range diskInfo.EcShardInfos {
  386. volumeIdToServer[ecShardInfo.Id] = VInfo{
  387. server: t.Id,
  388. collection: ecShardInfo.Collection,
  389. isEcVolume: true,
  390. }
  391. }
  392. }
  393. })
  394. if verbose {
  395. fmt.Fprintf(writer, "collected %d volumes and locations.\n", len(volumeIdToServer))
  396. }
  397. return
  398. }
  399. func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds []string, writer io.Writer) (err error) {
  400. fmt.Fprintf(writer, "purging orphan data for volume %d...\n", volumeId)
  401. locations, found := c.env.MasterClient.GetLocations(volumeId)
  402. if !found {
  403. return fmt.Errorf("failed to find volume %d locations", volumeId)
  404. }
  405. resultChan := make(chan []*volume_server_pb.DeleteResult, len(locations))
  406. var wg sync.WaitGroup
  407. for _, location := range locations {
  408. wg.Add(1)
  409. go func(server string, fidList []string) {
  410. defer wg.Done()
  411. if deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil {
  412. err = deleteErr
  413. } else if deleteResults != nil {
  414. resultChan <- deleteResults
  415. }
  416. }(location.Url, fileIds)
  417. }
  418. wg.Wait()
  419. close(resultChan)
  420. for results := range resultChan {
  421. for _, result := range results {
  422. if result.Error != "" {
  423. fmt.Fprintf(writer, "purge error: %s\n", result.Error)
  424. }
  425. }
  426. }
  427. return
  428. }
  429. func getVolumeFileIdFile(tempFolder string, vid uint32) string {
  430. return filepath.Join(tempFolder, fmt.Sprintf("%d.idx", vid))
  431. }
  432. func getFilerFileIdFile(tempFolder string, vid uint32) string {
  433. return filepath.Join(tempFolder, fmt.Sprintf("%d.fid", vid))
  434. }
  435. func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
  436. flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
  437. dst, err := os.OpenFile(fileName, flags, 0644)
  438. if err != nil {
  439. return nil
  440. }
  441. defer dst.Close()
  442. for {
  443. resp, receiveErr := client.Recv()
  444. if receiveErr == io.EOF {
  445. break
  446. }
  447. if receiveErr != nil {
  448. return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
  449. }
  450. dst.Write(resp.FileContent)
  451. }
  452. return nil
  453. }