Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

volume: atomic copying file, adds version and stopOffset

pull/934/head
Chris Lu 6 years ago
parent
730a032137
commit
3b3651dea3
9 changed files with 168 additions and 113 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      weed/command/backup.go
  2. 3
      weed/pb/volume_server.proto
  3. 197
      weed/pb/volume_server_pb/volume_server.pb.go
  4. 31
      weed/server/volume_grpc_copy.go
  5. 2
      weed/storage/store.go
  6. 2
      weed/storage/volume.go
  7. 2
      weed/storage/volume_backup.go
  8. 6
      weed/storage/volume_super_block.go
  9. 8
      weed/storage/volume_vacuum.go

4
weed/command/backup.go
View File

@ -90,7 +90,7 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) {
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
if err = v.Compact(0); err != nil {
fmt.Printf("Compact Volume before synchronizing %v\n", err)
return true
@ -99,7 +99,7 @@ func runBackup(cmd *Command, args []string) bool {
fmt.Printf("Commit Compact before synchronizing %v\n", err)
return true
}
v.SuperBlock.CompactRevision = uint16(stats.CompactRevision)
v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
}

3
weed/pb/volume_server.proto
View File

@ -162,6 +162,8 @@ message CopyFileRequest {
uint32 volume_id = 1;
bool is_idx_file = 2;
bool is_dat_file = 3;
uint32 compaction_revision = 4;
uint64 stop_offset = 5;
}
message CopyFileResponse {
bytes file_content = 1;
@ -188,6 +190,7 @@ message ReadVolumeFileStatusResponse {
uint64 dat_file_timestamp_seconds = 4;
uint64 dat_file_size = 5;
uint64 file_count = 6;
uint32 compaction_revision = 7;
}
message DiskStatus {

197
weed/pb/volume_server_pb/volume_server.pb.go
View File

@ -600,6 +600,8 @@ type CopyFileRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
IsIdxFile bool `protobuf:"varint,2,opt,name=is_idx_file,json=isIdxFile" json:"is_idx_file,omitempty"`
IsDatFile bool `protobuf:"varint,3,opt,name=is_dat_file,json=isDatFile" json:"is_dat_file,omitempty"`
CompactionRevision uint32 `protobuf:"varint,4,opt,name=compaction_revision,json=compactionRevision" json:"compaction_revision,omitempty"`
StopOffset uint64 `protobuf:"varint,5,opt,name=stop_offset,json=stopOffset" json:"stop_offset,omitempty"`
}
func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} }
@ -628,6 +630,20 @@ func (m *CopyFileRequest) GetIsDatFile() bool {
return false
}
func (m *CopyFileRequest) GetCompactionRevision() uint32 {
if m != nil {
return m.CompactionRevision
}
return 0
}
func (m *CopyFileRequest) GetStopOffset() uint64 {
if m != nil {
return m.StopOffset
}
return 0
}
type CopyFileResponse struct {
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
}
@ -731,6 +747,7 @@ type ReadVolumeFileStatusResponse struct {
DatFileTimestampSeconds uint64 `protobuf:"varint,4,opt,name=dat_file_timestamp_seconds,json=datFileTimestampSeconds" json:"dat_file_timestamp_seconds,omitempty"`
DatFileSize uint64 `protobuf:"varint,5,opt,name=dat_file_size,json=datFileSize" json:"dat_file_size,omitempty"`
FileCount uint64 `protobuf:"varint,6,opt,name=file_count,json=fileCount" json:"file_count,omitempty"`
CompactionRevision uint32 `protobuf:"varint,7,opt,name=compaction_revision,json=compactionRevision" json:"compaction_revision,omitempty"`
}
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
@ -780,6 +797,13 @@ func (m *ReadVolumeFileStatusResponse) GetFileCount() uint64 {
return 0
}
func (m *ReadVolumeFileStatusResponse) GetCompactionRevision() uint32 {
if m != nil {
return m.CompactionRevision
}
return 0
}
type DiskStatus struct {
Dir string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
All uint64 `protobuf:"varint,2,opt,name=all" json:"all,omitempty"`
@ -1579,89 +1603,92 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1339 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x58, 0xdd, 0x73, 0x14, 0x45,
0x10, 0xcf, 0x72, 0x97, 0xaf, 0xbe, 0x3b, 0x08, 0x13, 0x20, 0xc7, 0xf2, 0xe1, 0x31, 0x20, 0x1c,
0x1f, 0x06, 0x85, 0x52, 0x51, 0x1f, 0x14, 0x12, 0x2d, 0x53, 0x25, 0x50, 0xb5, 0x01, 0x4a, 0x0b,
0xab, 0xb6, 0x26, 0xbb, 0x93, 0x64, 0x2b, 0x7b, 0xb3, 0xcb, 0xce, 0x6c, 0x24, 0x96, 0xe5, 0x5f,
0xe2, 0x9b, 0xe5, 0x8b, 0x0f, 0xfe, 0x83, 0xbe, 0x58, 0xf3, 0xb1, 0x97, 0xfd, 0xcc, 0x2d, 0xf2,
0x36, 0xdb, 0xd3, 0xdd, 0xbf, 0xee, 0x9e, 0x9e, 0x9e, 0xdf, 0x1d, 0xac, 0x1e, 0x46, 0x61, 0x3a,
0xa1, 0x2e, 0xa7, 0xc9, 0x21, 0x4d, 0xd6, 0xe3, 0x24, 0x12, 0x11, 0x5a, 0x29, 0x08, 0xdd, 0x78,
0x07, 0xdf, 0x07, 0xf4, 0x84, 0x08, 0x6f, 0x7f, 0x93, 0x86, 0x54, 0x50, 0x87, 0xbe, 0x49, 0x29,
0x17, 0xe8, 0x22, 0x2c, 0xed, 0x06, 0x21, 0x75, 0x03, 0x9f, 0x0f, 0xad, 0x51, 0x67, 0xbc, 0xec,
0x2c, 0xca, 0xef, 0x2d, 0x9f, 0xe3, 0xe7, 0xb0, 0x5a, 0x30, 0xe0, 0x71, 0xc4, 0x38, 0x45, 0x8f,
0x60, 0x31, 0xa1, 0x3c, 0x0d, 0x85, 0x36, 0xe8, 0x3d, 0xb8, 0xba, 0x5e, 0xc6, 0x5a, 0x9f, 0x9a,
0xa4, 0xa1, 0x70, 0x32, 0x75, 0x1c, 0x40, 0x3f, 0xbf, 0x81, 0xd6, 0x60, 0xd1, 0x60, 0x0f, 0xad,
0x91, 0x35, 0x5e, 0x76, 0x16, 0x34, 0x34, 0xba, 0x00, 0x0b, 0x5c, 0x10, 0x91, 0xf2, 0xe1, 0xa9,
0x91, 0x35, 0x9e, 0x77, 0xcc, 0x17, 0x3a, 0x07, 0xf3, 0x34, 0x49, 0xa2, 0x64, 0xd8, 0x51, 0xea,
0xfa, 0x03, 0x21, 0xe8, 0xf2, 0xe0, 0x57, 0x3a, 0xec, 0x8e, 0xac, 0xf1, 0xc0, 0x51, 0x6b, 0xbc,
0x08, 0xf3, 0xdf, 0x4e, 0x62, 0x71, 0x84, 0x3f, 0x87, 0xe1, 0x2b, 0xe2, 0xa5, 0xe9, 0xe4, 0x95,
0x8a, 0x71, 0x63, 0x9f, 0x7a, 0x07, 0x59, 0xee, 0x97, 0x60, 0xd9, 0x44, 0x6e, 0x22, 0x18, 0x38,
0x4b, 0x5a, 0xb0, 0xe5, 0xe3, 0x6f, 0xe0, 0x62, 0x8d, 0xa1, 0xa9, 0xc1, 0x75, 0x18, 0xec, 0x91,
0x64, 0x87, 0xec, 0x51, 0x37, 0x21, 0x22, 0x88, 0x94, 0xb5, 0xe5, 0xf4, 0x8d, 0xd0, 0x91, 0x32,
0xfc, 0x1a, 0xec, 0x82, 0x87, 0x68, 0x12, 0x13, 0x4f, 0xb4, 0x01, 0x47, 0x23, 0xe8, 0xc5, 0x09,
0x25, 0x61, 0x18, 0x79, 0x44, 0x50, 0x55, 0x85, 0x8e, 0x93, 0x17, 0xe1, 0x2b, 0x70, 0xa9, 0xd6,
0xb9, 0x0e, 0x10, 0x3f, 0x2a, 0x45, 0x1f, 0x4d, 0x26, 0x41, 0x2b, 0x68, 0x7c, 0xb9, 0x12, 0xb5,
0xb2, 0x34, 0x7e, 0xbf, 0x28, 0xed, 0x86, 0x94, 0xb0, 0x34, 0x6e, 0xe5, 0xb8, 0x1c, 0x71, 0x66,
0x3a, 0xf5, 0xbc, 0xa6, 0x9b, 0x63, 0x23, 0x0a, 0x43, 0xea, 0x89, 0x20, 0x62, 0x99, 0xdb, 0xab,
0x00, 0xde, 0x54, 0x68, 0x5a, 0x25, 0x27, 0xc1, 0x36, 0x0c, 0xab, 0xa6, 0xc6, 0xed, 0xdf, 0x16,
0x9c, 0x7f, 0x6c, 0x8a, 0xa6, 0x81, 0x5b, 0x1d, 0x40, 0x11, 0xf2, 0x54, 0x19, 0xb2, 0x7c, 0x40,
0x9d, 0xca, 0x01, 0x49, 0x8d, 0x84, 0xc6, 0x61, 0xe0, 0x11, 0xe5, 0xa2, 0xab, 0x5c, 0xe4, 0x45,
0x68, 0x05, 0x3a, 0x42, 0x84, 0xc3, 0x79, 0xb5, 0x23, 0x97, 0x78, 0x08, 0x17, 0xca, 0xb1, 0x9a,
0x34, 0x3e, 0x83, 0x35, 0x2d, 0xd9, 0x3e, 0x62, 0xde, 0xb6, 0xba, 0x0d, 0xad, 0x8a, 0xfe, 0xaf,
0x05, 0xc3, 0xaa, 0xa1, 0xe9, 0xe2, 0xf7, 0xad, 0xc0, 0xbb, 0xe6, 0x87, 0x3e, 0x80, 0x9e, 0x20,
0x41, 0xe8, 0x46, 0xbb, 0xbb, 0x9c, 0x8a, 0xe1, 0xc2, 0xc8, 0x1a, 0x77, 0x1d, 0x90, 0xa2, 0xe7,
0x4a, 0x82, 0x6e, 0xc3, 0x8a, 0xa7, 0x3b, 0xd9, 0x4d, 0xe8, 0x61, 0xc0, 0xa5, 0xe7, 0x45, 0x15,
0xd8, 0x19, 0x2f, 0xeb, 0x70, 0x2d, 0x46, 0x18, 0x06, 0x81, 0xff, 0xd6, 0x55, 0x03, 0x44, 0x5d,
0xff, 0x25, 0xe5, 0xad, 0x17, 0xf8, 0x6f, 0xbf, 0x0b, 0x42, 0xba, 0x2d, 0xa7, 0xc0, 0x2b, 0xb8,
0xac, 0x93, 0xdf, 0x62, 0x5e, 0x42, 0x27, 0x94, 0x09, 0x12, 0x6e, 0x44, 0xf1, 0x51, 0xab, 0x16,
0xb8, 0x08, 0x4b, 0x3c, 0x60, 0x1e, 0x75, 0x99, 0x1e, 0x43, 0x5d, 0x67, 0x51, 0x7d, 0x3f, 0xe3,
0xf8, 0x09, 0x5c, 0x69, 0xf0, 0x6b, 0x2a, 0x7b, 0x0d, 0xfa, 0x2a, 0x30, 0x2f, 0x62, 0x82, 0x32,
0xa1, 0x7c, 0xf7, 0x9d, 0x9e, 0x94, 0x6d, 0x68, 0x11, 0xfe, 0x04, 0x90, 0xf6, 0xf1, 0x34, 0x4a,
0x59, 0xbb, 0xab, 0x79, 0x1e, 0x56, 0x0b, 0x26, 0xa6, 0x37, 0x1e, 0xc2, 0x39, 0x2d, 0x7e, 0xc9,
0x26, 0xad, 0x7d, 0xad, 0xc1, 0xf9, 0x92, 0x91, 0xf1, 0xf6, 0x20, 0x03, 0x29, 0xbe, 0x13, 0x27,
0x3a, 0xbb, 0x90, 0x45, 0x50, 0x7c, 0x2a, 0xf0, 0x3f, 0x16, 0x9c, 0xcd, 0xc6, 0x48, 0xcb, 0xaa,
0xbf, 0x63, 0xdb, 0x75, 0x1a, 0xdb, 0xae, 0x7b, 0xdc, 0x76, 0x63, 0x58, 0xe1, 0x51, 0x9a, 0x78,
0xd4, 0xf5, 0x89, 0x20, 0x2e, 0x8b, 0x7c, 0x6a, 0xba, 0xf2, 0xb4, 0x96, 0x6f, 0x12, 0x41, 0x9e,
0x45, 0x3e, 0xc5, 0x5f, 0x67, 0x87, 0x52, 0x38, 0xcd, 0xdb, 0x70, 0x36, 0x24, 0x5c, 0xb8, 0x24,
0x8e, 0x29, 0xf3, 0x5d, 0x22, 0x64, 0x4b, 0x58, 0xaa, 0x25, 0x4e, 0xcb, 0x8d, 0xc7, 0x4a, 0xfe,
0x58, 0x3c, 0xe3, 0x98, 0xc1, 0x19, 0x69, 0x2a, 0x3b, 0xb0, 0x65, 0xba, 0xbd, 0x80, 0xbb, 0x59,
0x23, 0xab, 0x7c, 0x97, 0x9c, 0xe5, 0x80, 0x6f, 0xe9, 0x2e, 0x36, 0xfb, 0x3e, 0x11, 0x7a, 0xbf,
0x93, 0xed, 0x6f, 0x12, 0x21, 0xf7, 0xf1, 0xa7, 0xb0, 0x72, 0x8c, 0xd7, 0xbe, 0xf9, 0xd2, 0xec,
0x5c, 0x5e, 0x90, 0x20, 0x7c, 0xcf, 0xdb, 0x80, 0xc6, 0x70, 0xc6, 0x4f, 0x48, 0xc0, 0x02, 0xb6,
0xb7, 0x4d, 0xbd, 0x88, 0xf9, 0x5c, 0xc5, 0x39, 0x70, 0xca, 0x62, 0xfc, 0x7b, 0x56, 0x5e, 0x0d,
0x7b, 0xfc, 0x98, 0x32, 0x4a, 0xfd, 0x90, 0xba, 0xfb, 0x94, 0xf8, 0x34, 0x31, 0x01, 0xf7, 0xb5,
0xf0, 0x7b, 0x25, 0x93, 0xa3, 0xc3, 0x28, 0xed, 0x44, 0xfe, 0x91, 0x0a, 0xa1, 0xef, 0x80, 0x16,
0x3d, 0x89, 0xfc, 0x23, 0x35, 0x0f, 0xb8, 0xab, 0xce, 0xc9, 0xdb, 0x4f, 0xd9, 0x81, 0xa9, 0x55,
0x2f, 0xe0, 0x3f, 0x10, 0x2e, 0x36, 0xa4, 0x08, 0x7f, 0x09, 0x97, 0x1c, 0x4a, 0x7c, 0x1d, 0x83,
0x9a, 0x12, 0xed, 0x27, 0xe9, 0x1f, 0xa7, 0xe0, 0x72, 0xbd, 0x71, 0x9b, 0x69, 0xfa, 0x15, 0xd8,
0xd3, 0x69, 0x25, 0x82, 0x09, 0xe5, 0x82, 0x4c, 0x62, 0x97, 0x9b, 0x72, 0xe9, 0x82, 0xae, 0x99,
0xd1, 0xf5, 0x22, 0xdb, 0x37, 0x65, 0xab, 0x8e, 0xba, 0x4e, 0x65, 0xd4, 0x49, 0x80, 0xac, 0x4b,
0x6a, 0x00, 0xba, 0x1a, 0xc0, 0xd7, 0x5d, 0x53, 0x07, 0x30, 0x35, 0x56, 0x00, 0xf3, 0x1a, 0xc0,
0xe8, 0x2b, 0x80, 0x2b, 0x00, 0xa6, 0xab, 0x52, 0x96, 0x8d, 0xee, 0x65, 0xdd, 0x53, 0x29, 0x13,
0xf8, 0x47, 0x80, 0xcd, 0x80, 0x1f, 0xe8, 0x9a, 0xc8, 0x3b, 0xe8, 0x07, 0x89, 0x79, 0xaa, 0xe5,
0x52, 0x4a, 0x48, 0x18, 0x9a, 0x4c, 0xe5, 0x52, 0xd2, 0xb6, 0x94, 0x53, 0xdf, 0x24, 0xa3, 0xd6,
0x52, 0xb6, 0x9b, 0x50, 0x6a, 0xe2, 0x55, 0x6b, 0xfc, 0xa7, 0x05, 0xcb, 0x4f, 0xe9, 0xc4, 0x78,
0xbe, 0x0a, 0xb0, 0x17, 0x25, 0x51, 0x2a, 0x02, 0x46, 0xf5, 0x25, 0x9c, 0x77, 0x72, 0x92, 0xff,
0x8f, 0xa3, 0x68, 0x24, 0x0d, 0x77, 0x4d, 0xee, 0x6a, 0x2d, 0x65, 0xfb, 0x94, 0xc4, 0x26, 0x5d,
0xb5, 0x96, 0x24, 0x94, 0x0b, 0xe2, 0x1d, 0xa8, 0x87, 0xa9, 0xeb, 0xe8, 0x8f, 0x07, 0x7f, 0x0d,
0xa0, 0x6f, 0x1e, 0x5a, 0xc5, 0x82, 0xd1, 0xcf, 0xd0, 0xcb, 0xb1, 0x67, 0x74, 0xa3, 0x4a, 0x92,
0xab, 0x6c, 0xdc, 0xfe, 0x70, 0x86, 0x96, 0x99, 0xab, 0x73, 0x88, 0xc1, 0xd9, 0x0a, 0x3b, 0x45,
0x77, 0xaa, 0xd6, 0x4d, 0xdc, 0xd7, 0xbe, 0xdb, 0x4a, 0x77, 0x8a, 0x27, 0x60, 0xb5, 0x86, 0x6e,
0xa2, 0x7b, 0x33, 0xbc, 0x14, 0x28, 0xaf, 0xfd, 0x51, 0x4b, 0xed, 0x29, 0xea, 0x1b, 0x40, 0x55,
0x2e, 0x8a, 0xee, 0xce, 0x74, 0x73, 0xcc, 0x75, 0xed, 0x7b, 0xed, 0x94, 0x1b, 0x13, 0xd5, 0x2c,
0x75, 0x66, 0xa2, 0x05, 0x1e, 0x3c, 0x33, 0xd1, 0x12, 0xf5, 0x9d, 0x43, 0x07, 0xb0, 0x52, 0x66,
0xb0, 0xe8, 0x76, 0xd3, 0xcf, 0xaa, 0x0a, 0x41, 0xb6, 0xef, 0xb4, 0x51, 0x9d, 0x82, 0x51, 0x38,
0x5d, 0x64, 0x99, 0xe8, 0x56, 0xd5, 0xbe, 0x96, 0x33, 0xdb, 0xe3, 0xd9, 0x8a, 0xf9, 0x9c, 0xca,
0xcc, 0xb3, 0x2e, 0xa7, 0x06, 0x5a, 0x5b, 0x97, 0x53, 0x13, 0x91, 0xc5, 0x73, 0xe8, 0xb7, 0x8c,
0xce, 0x94, 0x18, 0x19, 0x5a, 0x6f, 0x72, 0x53, 0x4f, 0x09, 0xed, 0xfb, 0xad, 0xf5, 0x33, 0xec,
0x8f, 0x2d, 0x79, 0xd7, 0x73, 0xc4, 0xac, 0xee, 0xae, 0x57, 0xa9, 0x5e, 0xdd, 0x5d, 0xaf, 0x63,
0x77, 0x73, 0x68, 0x07, 0x06, 0x05, 0xaa, 0x86, 0x6e, 0x36, 0x59, 0x16, 0x09, 0xa0, 0x7d, 0x6b,
0xa6, 0xde, 0x14, 0xc3, 0xcd, 0xa6, 0x97, 0x19, 0x57, 0x8d, 0xc1, 0x15, 0xe7, 0xd5, 0xcd, 0x59,
0x6a, 0x53, 0x80, 0x9f, 0x00, 0x8e, 0x99, 0x15, 0xba, 0xde, 0x64, 0x97, 0x3f, 0x8a, 0x1b, 0x27,
0x2b, 0x4d, 0x5d, 0xff, 0x02, 0xe7, 0xea, 0x1e, 0x66, 0x54, 0x73, 0x0b, 0x4f, 0x78, 0xfd, 0xed,
0xf5, 0xb6, 0xea, 0x53, 0xe0, 0x97, 0xb0, 0x94, 0x91, 0x2f, 0x74, 0xad, 0x6a, 0x5d, 0x22, 0x82,
0x36, 0x3e, 0x49, 0x25, 0xd7, 0x4d, 0xaf, 0xb3, 0x52, 0x49, 0x96, 0xd4, 0x5c, 0xaa, 0x1c, 0x75,
0x6b, 0x2e, 0x55, 0x9e, 0x68, 0x49, 0xe7, 0x3b, 0x0b, 0xea, 0xef, 0xa1, 0x87, 0xff, 0x05, 0x00,
0x00, 0xff, 0xff, 0xfc, 0x26, 0x3f, 0x74, 0x35, 0x12, 0x00, 0x00,
// 1378 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x58, 0xdb, 0x6e, 0xd4, 0x46,
0x18, 0x8e, 0xd9, 0xcd, 0xe9, 0xdf, 0x5d, 0x08, 0x13, 0x20, 0x8b, 0x39, 0x05, 0x43, 0x61, 0x39,
0x34, 0xb4, 0xa0, 0xb6, 0xb4, 0xbd, 0x68, 0x21, 0x69, 0xd5, 0x48, 0x05, 0x24, 0x07, 0x50, 0x2b,
0x2a, 0x59, 0x13, 0x7b, 0x92, 0x8c, 0xe2, 0xf5, 0x18, 0xcf, 0x98, 0x92, 0xaa, 0xea, 0xcb, 0x54,
0xbd, 0xe9, 0x45, 0x9f, 0xa1, 0x2f, 0xd0, 0x27, 0xea, 0x4d, 0x35, 0x07, 0x7b, 0xd7, 0xa7, 0xac,
0x29, 0x77, 0xe3, 0x6f, 0xfe, 0xf3, 0xfc, 0xf3, 0xcf, 0xb7, 0x0b, 0xab, 0x6f, 0x58, 0x98, 0x8e,
0x89, 0xc7, 0x49, 0xf2, 0x86, 0x24, 0x1b, 0x71, 0xc2, 0x04, 0x43, 0x2b, 0x05, 0xd0, 0x8b, 0x77,
0x9d, 0x7b, 0x80, 0x1e, 0x63, 0xe1, 0x1f, 0x6c, 0x91, 0x90, 0x08, 0xe2, 0x92, 0xd7, 0x29, 0xe1,
0x02, 0x9d, 0x87, 0xa5, 0x3d, 0x1a, 0x12, 0x8f, 0x06, 0x7c, 0x68, 0xad, 0x77, 0x46, 0xcb, 0xee,
0xa2, 0xfc, 0xde, 0x0e, 0xb8, 0xf3, 0x0c, 0x56, 0x0b, 0x0a, 0x3c, 0x66, 0x11, 0x27, 0xe8, 0x21,
0x2c, 0x26, 0x84, 0xa7, 0xa1, 0xd0, 0x0a, 0xbd, 0xfb, 0x97, 0x37, 0xca, 0xbe, 0x36, 0x72, 0x95,
0x34, 0x14, 0x6e, 0x26, 0xee, 0x50, 0xe8, 0x4f, 0x6f, 0xa0, 0x35, 0x58, 0x34, 0xbe, 0x87, 0xd6,
0xba, 0x35, 0x5a, 0x76, 0x17, 0xb4, 0x6b, 0x74, 0x0e, 0x16, 0xb8, 0xc0, 0x22, 0xe5, 0xc3, 0x13,
0xeb, 0xd6, 0x68, 0xde, 0x35, 0x5f, 0xe8, 0x0c, 0xcc, 0x93, 0x24, 0x61, 0xc9, 0xb0, 0xa3, 0xc4,
0xf5, 0x07, 0x42, 0xd0, 0xe5, 0xf4, 0x17, 0x32, 0xec, 0xae, 0x5b, 0xa3, 0x81, 0xab, 0xd6, 0xce,
0x22, 0xcc, 0x7f, 0x33, 0x8e, 0xc5, 0x91, 0xf3, 0x19, 0x0c, 0x5f, 0x62, 0x3f, 0x4d, 0xc7, 0x2f,
0x55, 0x8c, 0x9b, 0x07, 0xc4, 0x3f, 0xcc, 0x72, 0xbf, 0x00, 0xcb, 0x26, 0x72, 0x13, 0xc1, 0xc0,
0x5d, 0xd2, 0xc0, 0x76, 0xe0, 0x7c, 0x0d, 0xe7, 0x6b, 0x14, 0x4d, 0x0d, 0xae, 0xc1, 0x60, 0x1f,
0x27, 0xbb, 0x78, 0x9f, 0x78, 0x09, 0x16, 0x94, 0x29, 0x6d, 0xcb, 0xed, 0x1b, 0xd0, 0x95, 0x98,
0xf3, 0x0a, 0xec, 0x82, 0x05, 0x36, 0x8e, 0xb1, 0x2f, 0xda, 0x38, 0x47, 0xeb, 0xd0, 0x8b, 0x13,
0x82, 0xc3, 0x90, 0xf9, 0x58, 0x10, 0x55, 0x85, 0x8e, 0x3b, 0x0d, 0x39, 0x97, 0xe0, 0x42, 0xad,
0x71, 0x1d, 0xa0, 0xf3, 0xb0, 0x14, 0x3d, 0x1b, 0x8f, 0x69, 0x2b, 0xd7, 0xce, 0xc5, 0x4a, 0xd4,
0x4a, 0xd3, 0xd8, 0xfd, 0xbc, 0xb4, 0x1b, 0x12, 0x1c, 0xa5, 0x71, 0x2b, 0xc3, 0xe5, 0x88, 0x33,
0xd5, 0xdc, 0xf2, 0x9a, 0x6e, 0x8e, 0x4d, 0x16, 0x86, 0xc4, 0x17, 0x94, 0x45, 0x99, 0xd9, 0xcb,
0x00, 0x7e, 0x0e, 0x9a, 0x56, 0x99, 0x42, 0x1c, 0x1b, 0x86, 0x55, 0x55, 0x63, 0xf6, 0x4f, 0x0b,
0xce, 0x3e, 0x32, 0x45, 0xd3, 0x8e, 0x5b, 0x1d, 0x40, 0xd1, 0xe5, 0x89, 0xb2, 0xcb, 0xf2, 0x01,
0x75, 0x2a, 0x07, 0x24, 0x25, 0x12, 0x12, 0x87, 0xd4, 0xc7, 0xca, 0x44, 0x57, 0x99, 0x98, 0x86,
0xd0, 0x0a, 0x74, 0x84, 0x08, 0x87, 0xf3, 0x6a, 0x47, 0x2e, 0x9d, 0x21, 0x9c, 0x2b, 0xc7, 0x6a,
0xd2, 0xf8, 0x14, 0xd6, 0x34, 0xb2, 0x73, 0x14, 0xf9, 0x3b, 0xea, 0x36, 0xb4, 0x2a, 0xfa, 0xbf,
0x16, 0x0c, 0xab, 0x8a, 0xa6, 0x8b, 0xdf, 0xb7, 0x02, 0xef, 0x9a, 0x1f, 0xba, 0x02, 0x3d, 0x81,
0x69, 0xe8, 0xb1, 0xbd, 0x3d, 0x4e, 0xc4, 0x70, 0x61, 0xdd, 0x1a, 0x75, 0x5d, 0x90, 0xd0, 0x33,
0x85, 0xa0, 0x5b, 0xb0, 0xe2, 0xeb, 0x4e, 0xf6, 0x12, 0xf2, 0x86, 0x72, 0x69, 0x79, 0x51, 0x05,
0x76, 0xca, 0xcf, 0x3a, 0x5c, 0xc3, 0xc8, 0x81, 0x01, 0x0d, 0xde, 0x7a, 0x6a, 0x80, 0xa8, 0xeb,
0xbf, 0xa4, 0xac, 0xf5, 0x68, 0xf0, 0xf6, 0x5b, 0x1a, 0x92, 0x1d, 0x39, 0x05, 0x5e, 0xc2, 0x45,
0x9d, 0xfc, 0x76, 0xe4, 0x27, 0x64, 0x4c, 0x22, 0x81, 0xc3, 0x4d, 0x16, 0x1f, 0xb5, 0x6a, 0x81,
0xf3, 0xb0, 0xc4, 0x69, 0xe4, 0x13, 0x2f, 0xd2, 0x63, 0xa8, 0xeb, 0x2e, 0xaa, 0xef, 0xa7, 0xdc,
0x79, 0x0c, 0x97, 0x1a, 0xec, 0x9a, 0xca, 0x5e, 0x85, 0xbe, 0x0a, 0xcc, 0x67, 0x91, 0x20, 0x91,
0x50, 0xb6, 0xfb, 0x6e, 0x4f, 0x62, 0x9b, 0x1a, 0x72, 0x3e, 0x06, 0xa4, 0x6d, 0x3c, 0x61, 0x69,
0xd4, 0xee, 0x6a, 0x9e, 0x85, 0xd5, 0x82, 0x8a, 0xe9, 0x8d, 0x07, 0x70, 0x46, 0xc3, 0x2f, 0xa2,
0x71, 0x6b, 0x5b, 0x6b, 0x70, 0xb6, 0xa4, 0x64, 0xac, 0xdd, 0xcf, 0x9c, 0x14, 0xdf, 0x89, 0x63,
0x8d, 0x9d, 0xcb, 0x22, 0x28, 0x3e, 0x15, 0xce, 0x5f, 0x16, 0x9c, 0xce, 0xc6, 0x48, 0xcb, 0xaa,
0xbf, 0x63, 0xdb, 0x75, 0x1a, 0xdb, 0xae, 0x3b, 0x69, 0xbb, 0x11, 0xac, 0x70, 0x96, 0x26, 0x3e,
0xf1, 0x02, 0x2c, 0xb0, 0x17, 0xb1, 0x80, 0x98, 0xae, 0x3c, 0xa9, 0xf1, 0x2d, 0x2c, 0xf0, 0x53,
0x16, 0x10, 0xe7, 0xab, 0xec, 0x50, 0x0a, 0xa7, 0x79, 0x0b, 0x4e, 0x87, 0x98, 0x0b, 0x0f, 0xc7,
0x31, 0x89, 0x02, 0x0f, 0x0b, 0xd9, 0x12, 0x96, 0x6a, 0x89, 0x93, 0x72, 0xe3, 0x91, 0xc2, 0x1f,
0x89, 0xa7, 0xdc, 0xf9, 0xdb, 0x82, 0x53, 0x52, 0x57, 0xb6, 0x60, 0xcb, 0x7c, 0x7b, 0x94, 0x7b,
0x59, 0x27, 0xab, 0x84, 0x97, 0xdc, 0x65, 0xca, 0xb7, 0x75, 0x1b, 0x9b, 0xfd, 0x00, 0x0b, 0xbd,
0xdf, 0xc9, 0xf6, 0xb7, 0xb0, 0x50, 0xfb, 0xf7, 0x60, 0xd5, 0xdc, 0x0c, 0xca, 0xa2, 0xc9, 0xa5,
0xd1, 0x6f, 0x21, 0x9a, 0x6c, 0xe5, 0xf7, 0xe6, 0x0a, 0xf4, 0xb8, 0x60, 0x71, 0x76, 0x07, 0xe7,
0xf5, 0x1d, 0x94, 0x90, 0xbe, 0x83, 0xce, 0x27, 0xb0, 0x32, 0xc9, 0xa0, 0x7d, 0x3f, 0xa7, 0xd9,
0x51, 0x3f, 0xc7, 0x34, 0x7c, 0xcf, 0x0b, 0x86, 0x46, 0x70, 0x2a, 0x48, 0x30, 0x8d, 0x68, 0xb4,
0xbf, 0x43, 0x7c, 0x16, 0x05, 0x5c, 0x65, 0x3e, 0x70, 0xcb, 0xb0, 0xf3, 0x5b, 0x76, 0x62, 0xda,
0xed, 0xe4, 0x7d, 0x8e, 0x08, 0x09, 0x42, 0xe2, 0x1d, 0x10, 0x1c, 0x90, 0xc4, 0x04, 0xdc, 0xd7,
0xe0, 0x77, 0x0a, 0x93, 0x95, 0x30, 0x42, 0xbb, 0x2c, 0x38, 0x52, 0x21, 0xf4, 0x5d, 0xd0, 0xd0,
0x63, 0x16, 0x1c, 0xa9, 0x11, 0xc3, 0x3d, 0x75, 0xf4, 0xfe, 0x41, 0x1a, 0x1d, 0x9a, 0xea, 0xf7,
0x28, 0xff, 0x1e, 0x73, 0xb1, 0x29, 0x21, 0xe7, 0x0b, 0xb8, 0xe0, 0x12, 0x1c, 0xe8, 0x18, 0xd4,
0xe0, 0x69, 0x3f, 0x9c, 0xff, 0x39, 0x01, 0x17, 0xeb, 0x95, 0xdb, 0x0c, 0xe8, 0x2f, 0xc1, 0xce,
0x07, 0xa0, 0xa0, 0x63, 0xc2, 0x05, 0x1e, 0xc7, 0x1e, 0x37, 0xe5, 0xd2, 0x05, 0x5d, 0x33, 0xd3,
0xf0, 0x79, 0xb6, 0x6f, 0xca, 0x56, 0x9d, 0x9e, 0x9d, 0xca, 0xf4, 0x94, 0x0e, 0xb2, 0xbe, 0xab,
0x71, 0xd0, 0xd5, 0x0e, 0x02, 0xdd, 0x87, 0x75, 0x0e, 0x72, 0x65, 0xe5, 0x40, 0x37, 0x5a, 0xcf,
0xc8, 0x2b, 0x07, 0x97, 0x00, 0x4c, 0x57, 0xa5, 0x51, 0xf6, 0x1a, 0x2c, 0xeb, 0x9e, 0x4a, 0x23,
0xd1, 0xd4, 0xda, 0x8b, 0x4d, 0xad, 0xed, 0xfc, 0x00, 0xb0, 0x45, 0xf9, 0xa1, 0x2e, 0xa2, 0x9c,
0x03, 0x01, 0x4d, 0x0c, 0x5d, 0x90, 0x4b, 0x89, 0xe0, 0x30, 0x34, 0xa5, 0x91, 0x4b, 0x49, 0x1d,
0x53, 0x4e, 0x02, 0x93, 0xbd, 0x5a, 0x4b, 0x6c, 0x2f, 0x21, 0xc4, 0x24, 0xa8, 0xd6, 0xce, 0xef,
0x16, 0x2c, 0x3f, 0x21, 0x63, 0x63, 0xf9, 0x32, 0xc0, 0x3e, 0x4b, 0x58, 0x2a, 0x68, 0x44, 0xf4,
0x20, 0x98, 0x77, 0xa7, 0x90, 0xff, 0xef, 0x47, 0x51, 0x59, 0x12, 0xee, 0x99, 0x62, 0xa9, 0xb5,
0xc4, 0x0e, 0x08, 0x8e, 0x4d, 0x7d, 0xd4, 0x5a, 0x12, 0x61, 0x2e, 0xb0, 0x7f, 0xa8, 0x8a, 0xd1,
0x75, 0xf5, 0xc7, 0xfd, 0x3f, 0x06, 0xd0, 0x37, 0x8f, 0xbd, 0x62, 0xe2, 0xe8, 0x27, 0xe8, 0x4d,
0x31, 0x78, 0x74, 0xbd, 0x4a, 0xd4, 0xab, 0xbf, 0x08, 0xec, 0x0f, 0x66, 0x48, 0x99, 0xd9, 0x3e,
0x87, 0x22, 0x38, 0x5d, 0x61, 0xc8, 0xe8, 0x76, 0x55, 0xbb, 0x89, 0x7f, 0xdb, 0x77, 0x5a, 0xc9,
0xe6, 0xfe, 0x04, 0xac, 0xd6, 0x50, 0x5e, 0x74, 0x77, 0x86, 0x95, 0x02, 0xed, 0xb6, 0x3f, 0x6c,
0x29, 0x9d, 0x7b, 0x7d, 0x0d, 0xa8, 0xca, 0x87, 0xd1, 0x9d, 0x99, 0x66, 0x26, 0x7c, 0xdb, 0xbe,
0xdb, 0x4e, 0xb8, 0x31, 0x51, 0xcd, 0x94, 0x67, 0x26, 0x5a, 0xe0, 0xe2, 0x33, 0x13, 0x2d, 0xd1,
0xef, 0x39, 0x74, 0x08, 0x2b, 0x65, 0x16, 0x8d, 0x6e, 0x35, 0xfd, 0xb4, 0xab, 0x90, 0x74, 0xfb,
0x76, 0x1b, 0xd1, 0xdc, 0x19, 0x81, 0x93, 0x45, 0xa6, 0x8b, 0x6e, 0x56, 0xf5, 0x6b, 0x79, 0xbb,
0x3d, 0x9a, 0x2d, 0x38, 0x9d, 0x53, 0x99, 0xfd, 0xd6, 0xe5, 0xd4, 0x40, 0xad, 0xeb, 0x72, 0x6a,
0x22, 0xd3, 0xce, 0x1c, 0xfa, 0x35, 0xa3, 0x54, 0x25, 0x56, 0x88, 0x36, 0x9a, 0xcc, 0xd4, 0xd3,
0x52, 0xfb, 0x5e, 0x6b, 0xf9, 0xcc, 0xf7, 0x47, 0x96, 0xbc, 0xeb, 0x53, 0xe4, 0xb0, 0xee, 0xae,
0x57, 0xe9, 0x66, 0xdd, 0x5d, 0xaf, 0x63, 0x98, 0x73, 0x68, 0x17, 0x06, 0x05, 0xba, 0x88, 0x6e,
0x34, 0x69, 0x16, 0x49, 0xa8, 0x7d, 0x73, 0xa6, 0x5c, 0xee, 0xc3, 0xcb, 0xa6, 0x97, 0x19, 0x57,
0x8d, 0xc1, 0x15, 0xe7, 0xd5, 0x8d, 0x59, 0x62, 0xb9, 0x83, 0x1f, 0x01, 0x26, 0xec, 0x0e, 0x5d,
0x6b, 0xd2, 0x9b, 0x3e, 0x8a, 0xeb, 0xc7, 0x0b, 0xe5, 0xa6, 0x7f, 0x86, 0x33, 0x75, 0x2f, 0x39,
0xaa, 0xb9, 0x85, 0xc7, 0xd0, 0x05, 0x7b, 0xa3, 0xad, 0x78, 0xee, 0xf8, 0x05, 0x2c, 0x65, 0x6c,
0x0d, 0x5d, 0xad, 0x6a, 0x97, 0xb8, 0xa8, 0xed, 0x1c, 0x27, 0x32, 0xd5, 0x4d, 0xaf, 0xb2, 0x52,
0x49, 0x5a, 0xd5, 0x5c, 0xaa, 0x29, 0xae, 0xd7, 0x5c, 0xaa, 0x69, 0x66, 0x26, 0x8d, 0xef, 0x2e,
0xa8, 0xbf, 0xa8, 0x1e, 0xfc, 0x17, 0x00, 0x00, 0xff, 0xff, 0x07, 0x6f, 0xca, 0x8a, 0xb9, 0x12,
0x00, 0x00,
}

31
weed/server/volume_grpc_copy.go
View File

@ -52,9 +52,13 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
return fmt.Errorf("read volume file status failed, %v", err)
}
// println("source:", volFileInfoResp.String())
copyFileClient, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: req.VolumeId,
IsIdxFile: true,
CompactionRevision: volFileInfoResp.CompactionRevision,
StopOffset: volFileInfoResp.IdxFileSize,
})
if err != nil {
return fmt.Errorf("failed to start copying volume %d idx file: %v", req.VolumeId, err)
@ -68,6 +72,8 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
copyFileClient, err = client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: req.VolumeId,
IsDatFile: true,
CompactionRevision: volFileInfoResp.CompactionRevision,
StopOffset: volFileInfoResp.DatFileSize,
})
if err != nil {
return fmt.Errorf("failed to start copying volume %d dat file: %v", req.VolumeId, err)
@ -161,6 +167,7 @@ func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_se
resp.DatFileTimestampSeconds = uint64(modTime.Unix())
resp.IdxFileTimestampSeconds = uint64(modTime.Unix())
resp.FileCount = v.FileCount()
resp.CompactionRevision = uint32(v.CompactionRevision)
return resp, nil
}
@ -171,7 +178,13 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
return fmt.Errorf("not found volume id %d", req.VolumeId)
}
const BufferSize = 1024 * 16
if uint32(v.CompactionRevision) != req.CompactionRevision {
return fmt.Errorf("volume %d is compacted", req.VolumeId)
}
bytesToRead := int64(req.StopOffset)
const BufferSize = 1024 * 1024 * 2
var fileName = v.FileName()
if req.IsDatFile {
fileName += ".dat"
@ -186,19 +199,31 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
buffer := make([]byte, BufferSize)
for {
for bytesToRead > 0 {
bytesread, err := file.Read(buffer)
// println(fileName, "read", bytesread, "bytes, with target", bytesToRead)
if err != nil {
if err != io.EOF {
return err
}
// println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
break
}
stream.Send(&volume_server_pb.CopyFileResponse{
if int64(bytesread) > bytesToRead {
bytesread = int(bytesToRead)
}
err = stream.Send(&volume_server_pb.CopyFileResponse{
FileContent: buffer[:bytesread],
})
if err != nil {
// println("sending", bytesread, "bytes err", err.Error())
return err
}
bytesToRead -= int64(bytesread)
}

2
weed/storage/store.go
View File

@ -125,7 +125,7 @@ func (s *Store) Status() []*VolumeInfo {
DeletedByteCount: v.nm.DeletedSize(),
ReadOnly: v.readOnly,
Ttl: v.Ttl,
CompactRevision: uint32(v.CompactRevision),
CompactRevision: uint32(v.CompactionRevision),
}
stats = append(stats, s)
}

2
weed/storage/volume.go
View File

@ -166,6 +166,6 @@ func (v *Volume) ToVolumeInformationMessage() *master_pb.VolumeInformationMessag
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
Version: uint32(v.Version()),
Ttl: v.Ttl.ToUint32(),
CompactRevision: uint32(v.SuperBlock.CompactRevision),
CompactRevision: uint32(v.SuperBlock.CompactionRevision),
}
}

2
weed/storage/volume_backup.go
View File

@ -20,7 +20,7 @@ func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusRespons
}
syncStatus.Collection = v.Collection
syncStatus.IdxFileSize = v.nm.IndexFileSize()
syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
syncStatus.CompactRevision = uint32(v.SuperBlock.CompactionRevision)
syncStatus.Ttl = v.SuperBlock.Ttl.String()
syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
return syncStatus

6
weed/storage/volume_super_block.go
View File

@ -27,7 +27,7 @@ type SuperBlock struct {
version needle.Version
ReplicaPlacement *ReplicaPlacement
Ttl *needle.TTL
CompactRevision uint16
CompactionRevision uint16
Extra *master_pb.SuperBlockExtra
extraSize uint16
}
@ -48,7 +48,7 @@ func (s *SuperBlock) Bytes() []byte {
header[0] = byte(s.version)
header[1] = s.ReplicaPlacement.Byte()
s.Ttl.ToBytes(header[2:4])
util.Uint16toBytes(header[4:6], s.CompactRevision)
util.Uint16toBytes(header[4:6], s.CompactionRevision)
if s.Extra != nil {
extraData, err := proto.Marshal(s.Extra)
@ -112,7 +112,7 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
return
}
superBlock.Ttl = needle.LoadTTLFromBytes(header[2:4])
superBlock.CompactRevision = util.BytesToUint16(header[4:6])
superBlock.CompactionRevision = util.BytesToUint16(header[4:6])
superBlock.extraSize = util.BytesToUint16(header[6:8])
if superBlock.extraSize > 0 {

8
weed/storage/volume_vacuum.go
View File

@ -27,7 +27,7 @@ func (v *Volume) Compact(preallocate int64) error {
filePath := v.FileName()
v.lastCompactIndexOffset = v.nm.IndexFileSize()
v.lastCompactRevision = v.SuperBlock.CompactRevision
v.lastCompactRevision = v.SuperBlock.CompactionRevision
glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", preallocate)
}
@ -105,7 +105,7 @@ func fetchCompactRevisionFromDatFile(file *os.File) (compactRevision uint16, err
if err != nil {
return 0, err
}
return superBlock.CompactRevision, nil
return superBlock.CompactionRevision, nil
}
func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldIdxFileName string) (err error) {
@ -246,7 +246,7 @@ type VolumeFileScanner4Vacuum struct {
func (scanner *VolumeFileScanner4Vacuum) VisitSuperBlock(superBlock SuperBlock) error {
scanner.version = superBlock.Version()
superBlock.CompactRevision++
superBlock.CompactionRevision++
_, err := scanner.dst.Write(superBlock.Bytes())
scanner.newOffset = int64(superBlock.BlockSize())
return err
@ -321,7 +321,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
nm := NewBtreeNeedleMap(idx)
now := uint64(time.Now().Unix())
v.SuperBlock.CompactRevision++
v.SuperBlock.CompactionRevision++
dst.Write(v.SuperBlock.Bytes())
newOffset := int64(v.SuperBlock.BlockSize())

Write Preview
Loading…
Cancel
Save