|
@ -396,9 +396,8 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId |
|
|
} |
|
|
} |
|
|
buf.Write(resp.FileContent) |
|
|
buf.Write(resp.FileContent) |
|
|
} |
|
|
} |
|
|
fileredBuf := filterDeletedNeedleFromIdx(buf.Bytes()) |
|
|
|
|
|
if vinfo.isReadOnly == false { |
|
|
if vinfo.isReadOnly == false { |
|
|
index, err := idx.FirstInvalidIndex(fileredBuf.Bytes(), |
|
|
|
|
|
|
|
|
index, err := idx.FirstInvalidIndex(buf.Bytes(), |
|
|
func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { |
|
|
func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { |
|
|
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ |
|
|
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ |
|
|
VolumeId: volumeId, |
|
|
VolumeId: volumeId, |
|
@ -409,16 +408,16 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return false, fmt.Errorf("to read needle meta with id %d from volume %d with error %v", key, volumeId, err) |
|
|
return false, fmt.Errorf("to read needle meta with id %d from volume %d with error %v", key, volumeId, err) |
|
|
} |
|
|
} |
|
|
return resp.LastModified <= cutoffFrom, nil |
|
|
|
|
|
|
|
|
return resp.AppendAtNs <= cutoffFrom, nil |
|
|
}) |
|
|
}) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err) |
|
|
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err) |
|
|
} else { |
|
|
} else { |
|
|
fileredBuf.Truncate(index * types.NeedleMapEntrySize) |
|
|
|
|
|
|
|
|
buf.Truncate(index * types.NeedleMapEntrySize) |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
idxFilename := getVolumeFileIdFile(c.tempFolder, dataNodeId, volumeId) |
|
|
idxFilename := getVolumeFileIdFile(c.tempFolder, dataNodeId, volumeId) |
|
|
err = writeToFile(fileredBuf.Bytes(), idxFilename) |
|
|
|
|
|
|
|
|
err = writeToFile(buf.Bytes(), idxFilename) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) |
|
|
return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) |
|
|
} |
|
|
} |
|
@ -704,14 +703,3 @@ func writeToFile(bytes []byte, fileName string) error { |
|
|
dst.Write(bytes) |
|
|
dst.Write(bytes) |
|
|
return nil |
|
|
return nil |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func filterDeletedNeedleFromIdx(arr []byte) bytes.Buffer { |
|
|
|
|
|
var filteredBuf bytes.Buffer |
|
|
|
|
|
for i := 0; i < len(arr); i += types.NeedleMapEntrySize { |
|
|
|
|
|
size := types.BytesToSize(arr[i+types.NeedleIdSize+types.OffsetSize : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize]) |
|
|
|
|
|
if size > 0 { |
|
|
|
|
|
filteredBuf.Write(arr[i : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize]) |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
return filteredBuf |
|
|
|
|
|
} |
|
|
|