From 04b141749f5f549a6f88a528c3182a0584609f65 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 13 Aug 2025 00:57:36 -0700 Subject: [PATCH] explicit generation value --- BUILD_NOTES.md | 21 +++++++++++++ weed/pb/worker.proto | 2 ++ weed/pb/worker_pb/worker.pb.go | 30 ++++++++++++++++--- weed/worker/tasks/erasure_coding/detection.go | 1 + weed/worker/tasks/erasure_coding/ec_task.go | 2 ++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 BUILD_NOTES.md diff --git a/BUILD_NOTES.md b/BUILD_NOTES.md new file mode 100644 index 000000000..121759442 --- /dev/null +++ b/BUILD_NOTES.md @@ -0,0 +1,21 @@ +# Build Notes for SeaweedFS Development + +## Protobuf Generation + +To regenerate protobuf Go files after modifying .proto files: + +```bash +cd weed/pb +make +``` + +This will regenerate all the protobuf Go files with the latest changes from the .proto definitions. + +## Other Important Build Commands + +- Main build: `make` +- Clean build: `make clean && make` +- Tests: `make test` + +--- +*Generated: This file contains important build commands for development* diff --git a/weed/pb/worker.proto b/weed/pb/worker.proto index ede828743..1ef689764 100644 --- a/weed/pb/worker.proto +++ b/weed/pb/worker.proto @@ -134,6 +134,7 @@ message ErasureCodingTaskParams { string working_dir = 4; // Working directory for EC processing string master_client = 5; // Master server address bool cleanup_source = 6; // Whether to cleanup source volume after EC + uint32 generation = 7; // Generation number for EC shards (0=default, >0=generational) } // TaskSource represents a unified source location for any task type @@ -309,6 +310,7 @@ message ErasureCodingTaskConfig { int32 quiet_for_seconds = 2; // Minimum quiet time before EC int32 min_volume_size_mb = 3; // Minimum volume size for EC string collection_filter = 4; // Only process volumes from specific collections + uint32 generation = 5; // Generation number for EC shards (0=default, >0=generational) } // EcVacuumTaskConfig contains EC vacuum-specific configuration diff --git a/weed/pb/worker_pb/worker.pb.go b/weed/pb/worker_pb/worker.pb.go index e55ce6c5e..53bb33ced 100644 --- a/weed/pb/worker_pb/worker.pb.go +++ b/weed/pb/worker_pb/worker.pb.go @@ -1069,6 +1069,7 @@ type ErasureCodingTaskParams struct { WorkingDir string `protobuf:"bytes,4,opt,name=working_dir,json=workingDir,proto3" json:"working_dir,omitempty"` // Working directory for EC processing MasterClient string `protobuf:"bytes,5,opt,name=master_client,json=masterClient,proto3" json:"master_client,omitempty"` // Master server address CleanupSource bool `protobuf:"varint,6,opt,name=cleanup_source,json=cleanupSource,proto3" json:"cleanup_source,omitempty"` // Whether to cleanup source volume after EC + Generation uint32 `protobuf:"varint,7,opt,name=generation,proto3" json:"generation,omitempty"` // Generation number for EC shards (0=default, >0=generational) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1145,6 +1146,13 @@ func (x *ErasureCodingTaskParams) GetCleanupSource() bool { return false } +func (x *ErasureCodingTaskParams) GetGeneration() uint32 { + if x != nil { + return x.Generation + } + return 0 +} + // TaskSource represents a unified source location for any task type type TaskSource struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -2504,6 +2512,7 @@ type ErasureCodingTaskConfig struct { QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections + Generation uint32 `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"` // Generation number for EC shards (0=default, >0=generational) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2566,6 +2575,13 @@ func (x *ErasureCodingTaskConfig) GetCollectionFilter() string { return "" } +func (x *ErasureCodingTaskConfig) GetGeneration() uint32 { + if x != nil { + return x.Generation + } + return 0 +} + // EcVacuumTaskConfig contains EC vacuum-specific configuration type EcVacuumTaskConfig struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3388,7 +3404,7 @@ const file_worker_proto_rawDesc = "" + "batch_size\x18\x03 \x01(\x05R\tbatchSize\x12\x1f\n" + "\vworking_dir\x18\x04 \x01(\tR\n" + "workingDir\x12'\n" + - "\x0fverify_checksum\x18\x05 \x01(\bR\x0everifyChecksum\"\xfe\x01\n" + + "\x0fverify_checksum\x18\x05 \x01(\bR\x0everifyChecksum\"\x9e\x02\n" + "\x17ErasureCodingTaskParams\x120\n" + "\x14estimated_shard_size\x18\x01 \x01(\x04R\x12estimatedShardSize\x12\x1f\n" + "\vdata_shards\x18\x02 \x01(\x05R\n" + @@ -3397,7 +3413,10 @@ const file_worker_proto_rawDesc = "" + "\vworking_dir\x18\x04 \x01(\tR\n" + "workingDir\x12#\n" + "\rmaster_client\x18\x05 \x01(\tR\fmasterClient\x12%\n" + - "\x0ecleanup_source\x18\x06 \x01(\bR\rcleanupSource\"\xef\x01\n" + + "\x0ecleanup_source\x18\x06 \x01(\bR\rcleanupSource\x12\x1e\n" + + "\n" + + "generation\x18\a \x01(\rR\n" + + "generation\"\xef\x01\n" + "\n" + "TaskSource\x12\x12\n" + "\x04node\x18\x01 \x01(\tR\x04node\x12\x17\n" + @@ -3540,12 +3559,15 @@ const file_worker_proto_rawDesc = "" + "\x16check_interval_seconds\x18\x04 \x01(\x05R\x14checkIntervalSeconds\x12X\n" + "\x15erasure_coding_config\x18\x05 \x01(\v2\".worker_pb.ErasureCodingTaskConfigH\x00R\x13erasureCodingConfig\x12I\n" + "\x10ec_vacuum_config\x18\x06 \x01(\v2\x1d.worker_pb.EcVacuumTaskConfigH\x00R\x0eecVacuumConfigB\r\n" + - "\vtask_config\"\xc6\x01\n" + + "\vtask_config\"\xe6\x01\n" + "\x17ErasureCodingTaskConfig\x12%\n" + "\x0efullness_ratio\x18\x01 \x01(\x01R\rfullnessRatio\x12*\n" + "\x11quiet_for_seconds\x18\x02 \x01(\x05R\x0fquietForSeconds\x12+\n" + "\x12min_volume_size_mb\x18\x03 \x01(\x05R\x0fminVolumeSizeMb\x12+\n" + - "\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\"\xc5\x01\n" + + "\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\x12\x1e\n" + + "\n" + + "generation\x18\x05 \x01(\rR\n" + + "generation\"\xc5\x01\n" + "\x12EcVacuumTaskConfig\x12-\n" + "\x12deletion_threshold\x18\x01 \x01(\x01R\x11deletionThreshold\x123\n" + "\x16min_volume_age_seconds\x18\x02 \x01(\x05R\x13minVolumeAgeSeconds\x12+\n" + diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go index bb7c2c70f..6f827988a 100644 --- a/weed/worker/tasks/erasure_coding/detection.go +++ b/weed/worker/tasks/erasure_coding/detection.go @@ -425,6 +425,7 @@ func createECTaskParams(multiPlan *topology.MultiDestinationPlan) *worker_pb.Era return &worker_pb.ErasureCodingTaskParams{ DataShards: erasure_coding.DataShardsCount, // Standard data shards ParityShards: erasure_coding.ParityShardsCount, // Standard parity shards + Generation: 0, // Always use generation 0 for EC encoding } } diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go index a3a3d46dd..cb9b0e2c5 100644 --- a/weed/worker/tasks/erasure_coding/ec_task.go +++ b/weed/worker/tasks/erasure_coding/ec_task.go @@ -511,6 +511,7 @@ func (t *ErasureCodingTask) sendShardFileToDestination(destServer, filePath, sha IsEcVolume: true, ShardId: shardId, FileSize: uint64(fileInfo.Size()), + Generation: 0, // EC encoding always uses generation 0 }, }, }) @@ -590,6 +591,7 @@ func (t *ErasureCodingTask) mountEcShards() error { VolumeId: t.volumeID, Collection: t.collection, ShardIds: shardIds, + Generation: 0, // EC encoding always uses generation 0 }) return mountErr })