diff --git a/go.mod b/go.mod index cdf1bdf3f..106726e41 100644 --- a/go.mod +++ b/go.mod @@ -129,6 +129,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 github.com/cognusion/imaging v1.0.2 + github.com/container-storage-interface/spec v1.10.0 github.com/fluent/fluent-logger-golang v1.10.1 github.com/getsentry/sentry-go v0.42.0 github.com/go-ldap/ldap/v3 v3.4.12 @@ -138,7 +139,6 @@ require ( github.com/hashicorp/raft-boltdb/v2 v2.3.1 github.com/hashicorp/vault/api v1.22.0 github.com/jhump/protoreflect v1.18.0 - github.com/lib/pq v1.11.1 github.com/linkedin/goavro/v2 v2.14.1 github.com/mattn/go-sqlite3 v1.14.34 github.com/minio/crc64nvme v1.1.1 @@ -237,6 +237,7 @@ require ( github.com/klauspost/asmfmt v1.3.2 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.11.1 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/magiconair/properties v1.8.10 // indirect diff --git a/go.sum b/go.sum index 0d78f6de3..4965afea9 100644 --- a/go.sum +++ b/go.sum @@ -873,6 +873,8 @@ github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9 github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI= github.com/compose-spec/compose-go/v2 v2.6.0 h1:/+oBD2ixSENOeN/TlJqWZmUak0xM8A7J08w/z661Wd4= github.com/compose-spec/compose-go/v2 v2.6.0/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= +github.com/container-storage-interface/spec v1.10.0 h1:YkzWPV39x+ZMTa6Ax2czJLLwpryrQ+dPesB34mrRMXA= +github.com/container-storage-interface/spec v1.10.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc= github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= diff --git a/weed/pb/master.proto b/weed/pb/master.proto index a6a7b15e2..b27c768ce 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -57,6 +57,12 @@ service Seaweed { } rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { } + rpc CreateBlockVolume (CreateBlockVolumeRequest) returns (CreateBlockVolumeResponse) { + } + rpc DeleteBlockVolume (DeleteBlockVolumeRequest) returns (DeleteBlockVolumeResponse) { + } + rpc LookupBlockVolume (LookupBlockVolumeRequest) returns (LookupBlockVolumeResponse) { + } } ////////////////////////////////////////////////// @@ -89,6 +95,13 @@ message Heartbeat { // state flags volume_server_pb.VolumeServerState state = 23; + + // block volumes (full list on first heartbeat, empty thereafter) + repeated BlockVolumeInfoMessage block_volume_infos = 24; + // delta block volumes (subsequent heartbeats only) + repeated BlockVolumeShortInfoMessage new_block_volumes = 25; + repeated BlockVolumeShortInfoMessage deleted_block_volumes = 26; + bool has_no_block_volumes = 27; } message HeartbeatResponse { @@ -99,6 +112,8 @@ message HeartbeatResponse { repeated StorageBackend storage_backends = 5; repeated string duplicated_uuids = 6; bool preallocate = 7; + + repeated BlockVolumeAssignment block_volume_assignments = 10; } message VolumeInformationMessage { @@ -460,4 +475,63 @@ message RaftLeadershipTransferResponse { } message VolumeGrowResponse { +} + +////////////////////////////////////////////////// +// Block volume messages +////////////////////////////////////////////////// + +message BlockVolumeInfoMessage { + string path = 1; + uint64 volume_size = 2; + uint32 block_size = 3; + uint64 epoch = 4; + uint32 role = 5; + uint64 wal_head_lsn = 6; + uint64 checkpoint_lsn = 7; + bool has_lease = 8; + string disk_type = 9; +} + +message BlockVolumeShortInfoMessage { + string path = 1; + uint64 volume_size = 2; + uint32 block_size = 3; + string disk_type = 4; +} + +message BlockVolumeAssignment { + string path = 1; + uint64 epoch = 2; + uint32 role = 3; + uint32 lease_ttl_ms = 4; +} + +message CreateBlockVolumeRequest { + string name = 1; + uint64 size_bytes = 2; + string disk_type = 3; +} +message CreateBlockVolumeResponse { + string volume_id = 1; + string volume_server = 2; + string iscsi_addr = 3; + string iqn = 4; + uint64 capacity_bytes = 5; +} + +message DeleteBlockVolumeRequest { + string name = 1; +} +message DeleteBlockVolumeResponse { +} + +message LookupBlockVolumeRequest { + string name = 1; +} +message LookupBlockVolumeResponse { + string volume_server = 1; + string iscsi_addr = 2; + string iqn = 3; + uint64 capacity_bytes = 4; } \ No newline at end of file diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 27501ee8b..86ec5d1ba 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 -// protoc v6.33.4 +// protoc-gen-go v1.36.11 +// protoc v5.29.3 // source: master.proto package master_pb @@ -47,9 +47,15 @@ type Heartbeat struct { LocationUuids []string `protobuf:"bytes,21,rep,name=location_uuids,json=locationUuids,proto3" json:"location_uuids,omitempty"` Id string `protobuf:"bytes,22,opt,name=id,proto3" json:"id,omitempty"` // volume server id, independent of ip:port for stable identification // state flags - State *volume_server_pb.VolumeServerState `protobuf:"bytes,23,opt,name=state,proto3" json:"state,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + State *volume_server_pb.VolumeServerState `protobuf:"bytes,23,opt,name=state,proto3" json:"state,omitempty"` + // block volumes (full list on first heartbeat, empty thereafter) + BlockVolumeInfos []*BlockVolumeInfoMessage `protobuf:"bytes,24,rep,name=block_volume_infos,json=blockVolumeInfos,proto3" json:"block_volume_infos,omitempty"` + // delta block volumes (subsequent heartbeats only) + NewBlockVolumes []*BlockVolumeShortInfoMessage `protobuf:"bytes,25,rep,name=new_block_volumes,json=newBlockVolumes,proto3" json:"new_block_volumes,omitempty"` + DeletedBlockVolumes []*BlockVolumeShortInfoMessage `protobuf:"bytes,26,rep,name=deleted_block_volumes,json=deletedBlockVolumes,proto3" json:"deleted_block_volumes,omitempty"` + HasNoBlockVolumes bool `protobuf:"varint,27,opt,name=has_no_block_volumes,json=hasNoBlockVolumes,proto3" json:"has_no_block_volumes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Heartbeat) Reset() { @@ -222,15 +228,44 @@ func (x *Heartbeat) GetState() *volume_server_pb.VolumeServerState { return nil } +func (x *Heartbeat) GetBlockVolumeInfos() []*BlockVolumeInfoMessage { + if x != nil { + return x.BlockVolumeInfos + } + return nil +} + +func (x *Heartbeat) GetNewBlockVolumes() []*BlockVolumeShortInfoMessage { + if x != nil { + return x.NewBlockVolumes + } + return nil +} + +func (x *Heartbeat) GetDeletedBlockVolumes() []*BlockVolumeShortInfoMessage { + if x != nil { + return x.DeletedBlockVolumes + } + return nil +} + +func (x *Heartbeat) GetHasNoBlockVolumes() bool { + if x != nil { + return x.HasNoBlockVolumes + } + return false +} + type HeartbeatResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volume_size_limit,json=volumeSizeLimit,proto3" json:"volume_size_limit,omitempty"` - Leader string `protobuf:"bytes,2,opt,name=leader,proto3" json:"leader,omitempty"` - MetricsAddress string `protobuf:"bytes,3,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"` - MetricsIntervalSeconds uint32 `protobuf:"varint,4,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"` - StorageBackends []*StorageBackend `protobuf:"bytes,5,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"` - DuplicatedUuids []string `protobuf:"bytes,6,rep,name=duplicated_uuids,json=duplicatedUuids,proto3" json:"duplicated_uuids,omitempty"` - Preallocate bool `protobuf:"varint,7,opt,name=preallocate,proto3" json:"preallocate,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volume_size_limit,json=volumeSizeLimit,proto3" json:"volume_size_limit,omitempty"` + Leader string `protobuf:"bytes,2,opt,name=leader,proto3" json:"leader,omitempty"` + MetricsAddress string `protobuf:"bytes,3,opt,name=metrics_address,json=metricsAddress,proto3" json:"metrics_address,omitempty"` + MetricsIntervalSeconds uint32 `protobuf:"varint,4,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds,proto3" json:"metrics_interval_seconds,omitempty"` + StorageBackends []*StorageBackend `protobuf:"bytes,5,rep,name=storage_backends,json=storageBackends,proto3" json:"storage_backends,omitempty"` + DuplicatedUuids []string `protobuf:"bytes,6,rep,name=duplicated_uuids,json=duplicatedUuids,proto3" json:"duplicated_uuids,omitempty"` + Preallocate bool `protobuf:"varint,7,opt,name=preallocate,proto3" json:"preallocate,omitempty"` + BlockVolumeAssignments []*BlockVolumeAssignment `protobuf:"bytes,10,rep,name=block_volume_assignments,json=blockVolumeAssignments,proto3" json:"block_volume_assignments,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -314,6 +349,13 @@ func (x *HeartbeatResponse) GetPreallocate() bool { return false } +func (x *HeartbeatResponse) GetBlockVolumeAssignments() []*BlockVolumeAssignment { + if x != nil { + return x.BlockVolumeAssignments + } + return nil +} + type VolumeInformationMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` @@ -3840,6 +3882,578 @@ func (*VolumeGrowResponse) Descriptor() ([]byte, []int) { return file_master_proto_rawDescGZIP(), []int{60} } +type BlockVolumeInfoMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + VolumeSize uint64 `protobuf:"varint,2,opt,name=volume_size,json=volumeSize,proto3" json:"volume_size,omitempty"` + BlockSize uint32 `protobuf:"varint,3,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + Role uint32 `protobuf:"varint,5,opt,name=role,proto3" json:"role,omitempty"` + WalHeadLsn uint64 `protobuf:"varint,6,opt,name=wal_head_lsn,json=walHeadLsn,proto3" json:"wal_head_lsn,omitempty"` + CheckpointLsn uint64 `protobuf:"varint,7,opt,name=checkpoint_lsn,json=checkpointLsn,proto3" json:"checkpoint_lsn,omitempty"` + HasLease bool `protobuf:"varint,8,opt,name=has_lease,json=hasLease,proto3" json:"has_lease,omitempty"` + DiskType string `protobuf:"bytes,9,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockVolumeInfoMessage) Reset() { + *x = BlockVolumeInfoMessage{} + mi := &file_master_proto_msgTypes[61] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockVolumeInfoMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockVolumeInfoMessage) ProtoMessage() {} + +func (x *BlockVolumeInfoMessage) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[61] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockVolumeInfoMessage.ProtoReflect.Descriptor instead. +func (*BlockVolumeInfoMessage) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{61} +} + +func (x *BlockVolumeInfoMessage) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *BlockVolumeInfoMessage) GetVolumeSize() uint64 { + if x != nil { + return x.VolumeSize + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetBlockSize() uint32 { + if x != nil { + return x.BlockSize + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetRole() uint32 { + if x != nil { + return x.Role + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetWalHeadLsn() uint64 { + if x != nil { + return x.WalHeadLsn + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetCheckpointLsn() uint64 { + if x != nil { + return x.CheckpointLsn + } + return 0 +} + +func (x *BlockVolumeInfoMessage) GetHasLease() bool { + if x != nil { + return x.HasLease + } + return false +} + +func (x *BlockVolumeInfoMessage) GetDiskType() string { + if x != nil { + return x.DiskType + } + return "" +} + +type BlockVolumeShortInfoMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + VolumeSize uint64 `protobuf:"varint,2,opt,name=volume_size,json=volumeSize,proto3" json:"volume_size,omitempty"` + BlockSize uint32 `protobuf:"varint,3,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + DiskType string `protobuf:"bytes,4,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockVolumeShortInfoMessage) Reset() { + *x = BlockVolumeShortInfoMessage{} + mi := &file_master_proto_msgTypes[62] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockVolumeShortInfoMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockVolumeShortInfoMessage) ProtoMessage() {} + +func (x *BlockVolumeShortInfoMessage) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[62] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockVolumeShortInfoMessage.ProtoReflect.Descriptor instead. +func (*BlockVolumeShortInfoMessage) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{62} +} + +func (x *BlockVolumeShortInfoMessage) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *BlockVolumeShortInfoMessage) GetVolumeSize() uint64 { + if x != nil { + return x.VolumeSize + } + return 0 +} + +func (x *BlockVolumeShortInfoMessage) GetBlockSize() uint32 { + if x != nil { + return x.BlockSize + } + return 0 +} + +func (x *BlockVolumeShortInfoMessage) GetDiskType() string { + if x != nil { + return x.DiskType + } + return "" +} + +type BlockVolumeAssignment struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Role uint32 `protobuf:"varint,3,opt,name=role,proto3" json:"role,omitempty"` + LeaseTtlMs uint32 `protobuf:"varint,4,opt,name=lease_ttl_ms,json=leaseTtlMs,proto3" json:"lease_ttl_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockVolumeAssignment) Reset() { + *x = BlockVolumeAssignment{} + mi := &file_master_proto_msgTypes[63] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockVolumeAssignment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockVolumeAssignment) ProtoMessage() {} + +func (x *BlockVolumeAssignment) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[63] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockVolumeAssignment.ProtoReflect.Descriptor instead. +func (*BlockVolumeAssignment) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{63} +} + +func (x *BlockVolumeAssignment) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *BlockVolumeAssignment) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *BlockVolumeAssignment) GetRole() uint32 { + if x != nil { + return x.Role + } + return 0 +} + +func (x *BlockVolumeAssignment) GetLeaseTtlMs() uint32 { + if x != nil { + return x.LeaseTtlMs + } + return 0 +} + +type CreateBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBlockVolumeRequest) Reset() { + *x = CreateBlockVolumeRequest{} + mi := &file_master_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBlockVolumeRequest) ProtoMessage() {} + +func (x *CreateBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[64] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*CreateBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{64} +} + +func (x *CreateBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateBlockVolumeRequest) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *CreateBlockVolumeRequest) GetDiskType() string { + if x != nil { + return x.DiskType + } + return "" +} + +type CreateBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + VolumeServer string `protobuf:"bytes,2,opt,name=volume_server,json=volumeServer,proto3" json:"volume_server,omitempty"` + IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` + Iqn string `protobuf:"bytes,4,opt,name=iqn,proto3" json:"iqn,omitempty"` + CapacityBytes uint64 `protobuf:"varint,5,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBlockVolumeResponse) Reset() { + *x = CreateBlockVolumeResponse{} + mi := &file_master_proto_msgTypes[65] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBlockVolumeResponse) ProtoMessage() {} + +func (x *CreateBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[65] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*CreateBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{65} +} + +func (x *CreateBlockVolumeResponse) GetVolumeId() string { + if x != nil { + return x.VolumeId + } + return "" +} + +func (x *CreateBlockVolumeResponse) GetVolumeServer() string { + if x != nil { + return x.VolumeServer + } + return "" +} + +func (x *CreateBlockVolumeResponse) GetIscsiAddr() string { + if x != nil { + return x.IscsiAddr + } + return "" +} + +func (x *CreateBlockVolumeResponse) GetIqn() string { + if x != nil { + return x.Iqn + } + return "" +} + +func (x *CreateBlockVolumeResponse) GetCapacityBytes() uint64 { + if x != nil { + return x.CapacityBytes + } + return 0 +} + +type DeleteBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockVolumeRequest) Reset() { + *x = DeleteBlockVolumeRequest{} + mi := &file_master_proto_msgTypes[66] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockVolumeRequest) ProtoMessage() {} + +func (x *DeleteBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[66] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*DeleteBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{66} +} + +func (x *DeleteBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type DeleteBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBlockVolumeResponse) Reset() { + *x = DeleteBlockVolumeResponse{} + mi := &file_master_proto_msgTypes[67] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBlockVolumeResponse) ProtoMessage() {} + +func (x *DeleteBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[67] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*DeleteBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{67} +} + +type LookupBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupBlockVolumeRequest) Reset() { + *x = LookupBlockVolumeRequest{} + mi := &file_master_proto_msgTypes[68] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupBlockVolumeRequest) ProtoMessage() {} + +func (x *LookupBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[68] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*LookupBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{68} +} + +func (x *LookupBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type LookupBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeServer string `protobuf:"bytes,1,opt,name=volume_server,json=volumeServer,proto3" json:"volume_server,omitempty"` + IscsiAddr string `protobuf:"bytes,2,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` + Iqn string `protobuf:"bytes,3,opt,name=iqn,proto3" json:"iqn,omitempty"` + CapacityBytes uint64 `protobuf:"varint,4,opt,name=capacity_bytes,json=capacityBytes,proto3" json:"capacity_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupBlockVolumeResponse) Reset() { + *x = LookupBlockVolumeResponse{} + mi := &file_master_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupBlockVolumeResponse) ProtoMessage() {} + +func (x *LookupBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_master_proto_msgTypes[69] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*LookupBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_master_proto_rawDescGZIP(), []int{69} +} + +func (x *LookupBlockVolumeResponse) GetVolumeServer() string { + if x != nil { + return x.VolumeServer + } + return "" +} + +func (x *LookupBlockVolumeResponse) GetIscsiAddr() string { + if x != nil { + return x.IscsiAddr + } + return "" +} + +func (x *LookupBlockVolumeResponse) GetIqn() string { + if x != nil { + return x.Iqn + } + return "" +} + +func (x *LookupBlockVolumeResponse) GetCapacityBytes() uint64 { + if x != nil { + return x.CapacityBytes + } + return 0 +} + type SuperBlockExtra_ErasureCoding struct { state protoimpl.MessageState `protogen:"open.v1"` Data uint32 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` @@ -3851,7 +4465,7 @@ type SuperBlockExtra_ErasureCoding struct { func (x *SuperBlockExtra_ErasureCoding) Reset() { *x = SuperBlockExtra_ErasureCoding{} - mi := &file_master_proto_msgTypes[63] + mi := &file_master_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3863,7 +4477,7 @@ func (x *SuperBlockExtra_ErasureCoding) String() string { func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {} func (x *SuperBlockExtra_ErasureCoding) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[63] + mi := &file_master_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3912,7 +4526,7 @@ type LookupVolumeResponse_VolumeIdLocation struct { func (x *LookupVolumeResponse_VolumeIdLocation) Reset() { *x = LookupVolumeResponse_VolumeIdLocation{} - mi := &file_master_proto_msgTypes[64] + mi := &file_master_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3924,7 +4538,7 @@ func (x *LookupVolumeResponse_VolumeIdLocation) String() string { func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {} func (x *LookupVolumeResponse_VolumeIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[64] + mi := &file_master_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3978,7 +4592,7 @@ type LookupEcVolumeResponse_EcShardIdLocation struct { func (x *LookupEcVolumeResponse_EcShardIdLocation) Reset() { *x = LookupEcVolumeResponse_EcShardIdLocation{} - mi := &file_master_proto_msgTypes[69] + mi := &file_master_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3990,7 +4604,7 @@ func (x *LookupEcVolumeResponse_EcShardIdLocation) String() string { func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage() {} func (x *LookupEcVolumeResponse_EcShardIdLocation) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[69] + mi := &file_master_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4033,7 +4647,7 @@ type ListClusterNodesResponse_ClusterNode struct { func (x *ListClusterNodesResponse_ClusterNode) Reset() { *x = ListClusterNodesResponse_ClusterNode{} - mi := &file_master_proto_msgTypes[70] + mi := &file_master_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4045,7 +4659,7 @@ func (x *ListClusterNodesResponse_ClusterNode) String() string { func (*ListClusterNodesResponse_ClusterNode) ProtoMessage() {} func (x *ListClusterNodesResponse_ClusterNode) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[70] + mi := &file_master_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4108,7 +4722,7 @@ type RaftListClusterServersResponse_ClusterServers struct { func (x *RaftListClusterServersResponse_ClusterServers) Reset() { *x = RaftListClusterServersResponse_ClusterServers{} - mi := &file_master_proto_msgTypes[71] + mi := &file_master_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4120,7 +4734,7 @@ func (x *RaftListClusterServersResponse_ClusterServers) String() string { func (*RaftListClusterServersResponse_ClusterServers) ProtoMessage() {} func (x *RaftListClusterServersResponse_ClusterServers) ProtoReflect() protoreflect.Message { - mi := &file_master_proto_msgTypes[71] + mi := &file_master_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4168,7 +4782,8 @@ var File_master_proto protoreflect.FileDescriptor const file_master_proto_rawDesc = "" + "\n" + - "\fmaster.proto\x12\tmaster_pb\x1a\x13volume_server.proto\"\x8b\b\n" + + "\fmaster.proto\x12\tmaster_pb\x1a\x13volume_server.proto\"\xbd\n" + + "\n" + "\tHeartbeat\x12\x0e\n" + "\x02ip\x18\x01 \x01(\tR\x02ip\x12\x12\n" + "\x04port\x18\x02 \x01(\rR\x04port\x12\x1d\n" + @@ -4195,10 +4810,14 @@ const file_master_proto_rawDesc = "" + "\tgrpc_port\x18\x14 \x01(\rR\bgrpcPort\x12%\n" + "\x0elocation_uuids\x18\x15 \x03(\tR\rlocationUuids\x12\x0e\n" + "\x02id\x18\x16 \x01(\tR\x02id\x129\n" + - "\x05state\x18\x17 \x01(\v2#.volume_server_pb.VolumeServerStateR\x05state\x1aB\n" + + "\x05state\x18\x17 \x01(\v2#.volume_server_pb.VolumeServerStateR\x05state\x12O\n" + + "\x12block_volume_infos\x18\x18 \x03(\v2!.master_pb.BlockVolumeInfoMessageR\x10blockVolumeInfos\x12R\n" + + "\x11new_block_volumes\x18\x19 \x03(\v2&.master_pb.BlockVolumeShortInfoMessageR\x0fnewBlockVolumes\x12Z\n" + + "\x15deleted_block_volumes\x18\x1a \x03(\v2&.master_pb.BlockVolumeShortInfoMessageR\x13deletedBlockVolumes\x12/\n" + + "\x14has_no_block_volumes\x18\x1b \x01(\bR\x11hasNoBlockVolumes\x1aB\n" + "\x14MaxVolumeCountsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\rR\x05value:\x028\x01\"\xcd\x02\n" + + "\x05value\x18\x02 \x01(\rR\x05value:\x028\x01\"\xa9\x03\n" + "\x11HeartbeatResponse\x12*\n" + "\x11volume_size_limit\x18\x01 \x01(\x04R\x0fvolumeSizeLimit\x12\x16\n" + "\x06leader\x18\x02 \x01(\tR\x06leader\x12'\n" + @@ -4206,7 +4825,9 @@ const file_master_proto_rawDesc = "" + "\x18metrics_interval_seconds\x18\x04 \x01(\rR\x16metricsIntervalSeconds\x12D\n" + "\x10storage_backends\x18\x05 \x03(\v2\x19.master_pb.StorageBackendR\x0fstorageBackends\x12)\n" + "\x10duplicated_uuids\x18\x06 \x03(\tR\x0fduplicatedUuids\x12 \n" + - "\vpreallocate\x18\a \x01(\bR\vpreallocate\"\xb1\x04\n" + + "\vpreallocate\x18\a \x01(\bR\vpreallocate\x12Z\n" + + "\x18block_volume_assignments\x18\n" + + " \x03(\v2 .master_pb.BlockVolumeAssignmentR\x16blockVolumeAssignments\"\xb1\x04\n" + "\x18VolumeInformationMessage\x12\x0e\n" + "\x02id\x18\x01 \x01(\rR\x02id\x12\x12\n" + "\x04size\x18\x02 \x01(\x04R\x04size\x12\x1e\n" + @@ -4528,7 +5149,56 @@ const file_master_proto_rawDesc = "" + "\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" + "\n" + "new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" + - "\x12VolumeGrowResponse2\xc6\x10\n" + + "\x12VolumeGrowResponse\"\x99\x02\n" + + "\x16BlockVolumeInfoMessage\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + + "\vvolume_size\x18\x02 \x01(\x04R\n" + + "volumeSize\x12\x1d\n" + + "\n" + + "block_size\x18\x03 \x01(\rR\tblockSize\x12\x14\n" + + "\x05epoch\x18\x04 \x01(\x04R\x05epoch\x12\x12\n" + + "\x04role\x18\x05 \x01(\rR\x04role\x12 \n" + + "\fwal_head_lsn\x18\x06 \x01(\x04R\n" + + "walHeadLsn\x12%\n" + + "\x0echeckpoint_lsn\x18\a \x01(\x04R\rcheckpointLsn\x12\x1b\n" + + "\thas_lease\x18\b \x01(\bR\bhasLease\x12\x1b\n" + + "\tdisk_type\x18\t \x01(\tR\bdiskType\"\x8e\x01\n" + + "\x1bBlockVolumeShortInfoMessage\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + + "\vvolume_size\x18\x02 \x01(\x04R\n" + + "volumeSize\x12\x1d\n" + + "\n" + + "block_size\x18\x03 \x01(\rR\tblockSize\x12\x1b\n" + + "\tdisk_type\x18\x04 \x01(\tR\bdiskType\"w\n" + + "\x15BlockVolumeAssignment\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" + + "\x05epoch\x18\x02 \x01(\x04R\x05epoch\x12\x12\n" + + "\x04role\x18\x03 \x01(\rR\x04role\x12 \n" + + "\flease_ttl_ms\x18\x04 \x01(\rR\n" + + "leaseTtlMs\"j\n" + + "\x18CreateBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\n" + + "size_bytes\x18\x02 \x01(\x04R\tsizeBytes\x12\x1b\n" + + "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"\xb5\x01\n" + + "\x19CreateBlockVolumeResponse\x12\x1b\n" + + "\tvolume_id\x18\x01 \x01(\tR\bvolumeId\x12#\n" + + "\rvolume_server\x18\x02 \x01(\tR\fvolumeServer\x12\x1d\n" + + "\n" + + "iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\x12\x10\n" + + "\x03iqn\x18\x04 \x01(\tR\x03iqn\x12%\n" + + "\x0ecapacity_bytes\x18\x05 \x01(\x04R\rcapacityBytes\".\n" + + "\x18DeleteBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\x1b\n" + + "\x19DeleteBlockVolumeResponse\".\n" + + "\x18LookupBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\x98\x01\n" + + "\x19LookupBlockVolumeResponse\x12#\n" + + "\rvolume_server\x18\x01 \x01(\tR\fvolumeServer\x12\x1d\n" + + "\n" + + "iscsi_addr\x18\x02 \x01(\tR\tiscsiAddr\x12\x10\n" + + "\x03iqn\x18\x03 \x01(\tR\x03iqn\x12%\n" + + "\x0ecapacity_bytes\x18\x04 \x01(\x04R\rcapacityBytes2\xec\x12\n" + "\aSeaweed\x12I\n" + "\rSendHeartbeat\x12\x14.master_pb.Heartbeat\x1a\x1c.master_pb.HeartbeatResponse\"\x00(\x010\x01\x12X\n" + "\rKeepConnected\x12\x1f.master_pb.KeepConnectedRequest\x1a .master_pb.KeepConnectedResponse\"\x00(\x010\x01\x12Q\n" + @@ -4556,7 +5226,10 @@ const file_master_proto_rawDesc = "" + "\x10RaftRemoveServer\x12\".master_pb.RaftRemoveServerRequest\x1a#.master_pb.RaftRemoveServerResponse\"\x00\x12o\n" + "\x16RaftLeadershipTransfer\x12(.master_pb.RaftLeadershipTransferRequest\x1a).master_pb.RaftLeadershipTransferResponse\"\x00\x12K\n" + "\n" + - "VolumeGrow\x12\x1c.master_pb.VolumeGrowRequest\x1a\x1d.master_pb.VolumeGrowResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" + "VolumeGrow\x12\x1c.master_pb.VolumeGrowRequest\x1a\x1d.master_pb.VolumeGrowResponse\"\x00\x12`\n" + + "\x11CreateBlockVolume\x12#.master_pb.CreateBlockVolumeRequest\x1a$.master_pb.CreateBlockVolumeResponse\"\x00\x12`\n" + + "\x11DeleteBlockVolume\x12#.master_pb.DeleteBlockVolumeRequest\x1a$.master_pb.DeleteBlockVolumeResponse\"\x00\x12`\n" + + "\x11LookupBlockVolume\x12#.master_pb.LookupBlockVolumeRequest\x1a$.master_pb.LookupBlockVolumeResponse\"\x00B2Z0github.com/seaweedfs/seaweedfs/weed/pb/master_pbb\x06proto3" var ( file_master_proto_rawDescOnce sync.Once @@ -4570,7 +5243,7 @@ func file_master_proto_rawDescGZIP() []byte { return file_master_proto_rawDescData } -var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 72) +var file_master_proto_msgTypes = make([]protoimpl.MessageInfo, 81) var file_master_proto_goTypes = []any{ (*Heartbeat)(nil), // 0: master_pb.Heartbeat (*HeartbeatResponse)(nil), // 1: master_pb.HeartbeatResponse @@ -4633,18 +5306,27 @@ var file_master_proto_goTypes = []any{ (*RaftLeadershipTransferRequest)(nil), // 58: master_pb.RaftLeadershipTransferRequest (*RaftLeadershipTransferResponse)(nil), // 59: master_pb.RaftLeadershipTransferResponse (*VolumeGrowResponse)(nil), // 60: master_pb.VolumeGrowResponse - nil, // 61: master_pb.Heartbeat.MaxVolumeCountsEntry - nil, // 62: master_pb.StorageBackend.PropertiesEntry - (*SuperBlockExtra_ErasureCoding)(nil), // 63: master_pb.SuperBlockExtra.ErasureCoding - (*LookupVolumeResponse_VolumeIdLocation)(nil), // 64: master_pb.LookupVolumeResponse.VolumeIdLocation - nil, // 65: master_pb.DataNodeInfo.DiskInfosEntry - nil, // 66: master_pb.RackInfo.DiskInfosEntry - nil, // 67: master_pb.DataCenterInfo.DiskInfosEntry - nil, // 68: master_pb.TopologyInfo.DiskInfosEntry - (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 69: master_pb.LookupEcVolumeResponse.EcShardIdLocation - (*ListClusterNodesResponse_ClusterNode)(nil), // 70: master_pb.ListClusterNodesResponse.ClusterNode - (*RaftListClusterServersResponse_ClusterServers)(nil), // 71: master_pb.RaftListClusterServersResponse.ClusterServers - (*volume_server_pb.VolumeServerState)(nil), // 72: volume_server_pb.VolumeServerState + (*BlockVolumeInfoMessage)(nil), // 61: master_pb.BlockVolumeInfoMessage + (*BlockVolumeShortInfoMessage)(nil), // 62: master_pb.BlockVolumeShortInfoMessage + (*BlockVolumeAssignment)(nil), // 63: master_pb.BlockVolumeAssignment + (*CreateBlockVolumeRequest)(nil), // 64: master_pb.CreateBlockVolumeRequest + (*CreateBlockVolumeResponse)(nil), // 65: master_pb.CreateBlockVolumeResponse + (*DeleteBlockVolumeRequest)(nil), // 66: master_pb.DeleteBlockVolumeRequest + (*DeleteBlockVolumeResponse)(nil), // 67: master_pb.DeleteBlockVolumeResponse + (*LookupBlockVolumeRequest)(nil), // 68: master_pb.LookupBlockVolumeRequest + (*LookupBlockVolumeResponse)(nil), // 69: master_pb.LookupBlockVolumeResponse + nil, // 70: master_pb.Heartbeat.MaxVolumeCountsEntry + nil, // 71: master_pb.StorageBackend.PropertiesEntry + (*SuperBlockExtra_ErasureCoding)(nil), // 72: master_pb.SuperBlockExtra.ErasureCoding + (*LookupVolumeResponse_VolumeIdLocation)(nil), // 73: master_pb.LookupVolumeResponse.VolumeIdLocation + nil, // 74: master_pb.DataNodeInfo.DiskInfosEntry + nil, // 75: master_pb.RackInfo.DiskInfosEntry + nil, // 76: master_pb.DataCenterInfo.DiskInfosEntry + nil, // 77: master_pb.TopologyInfo.DiskInfosEntry + (*LookupEcVolumeResponse_EcShardIdLocation)(nil), // 78: master_pb.LookupEcVolumeResponse.EcShardIdLocation + (*ListClusterNodesResponse_ClusterNode)(nil), // 79: master_pb.ListClusterNodesResponse.ClusterNode + (*RaftListClusterServersResponse_ClusterServers)(nil), // 80: master_pb.RaftListClusterServersResponse.ClusterServers + (*volume_server_pb.VolumeServerState)(nil), // 81: volume_server_pb.VolumeServerState } var file_master_proto_depIdxs = []int32{ 2, // 0: master_pb.Heartbeat.volumes:type_name -> master_pb.VolumeInformationMessage @@ -4653,90 +5335,100 @@ var file_master_proto_depIdxs = []int32{ 4, // 3: master_pb.Heartbeat.ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 4: master_pb.Heartbeat.new_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage 4, // 5: master_pb.Heartbeat.deleted_ec_shards:type_name -> master_pb.VolumeEcShardInformationMessage - 61, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry - 72, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState - 5, // 8: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend - 62, // 9: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry - 63, // 10: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding - 9, // 11: master_pb.KeepConnectedResponse.volume_location:type_name -> master_pb.VolumeLocation - 10, // 12: master_pb.KeepConnectedResponse.cluster_node_update:type_name -> master_pb.ClusterNodeUpdate - 64, // 13: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation - 14, // 14: master_pb.AssignResponse.replicas:type_name -> master_pb.Location - 14, // 15: master_pb.AssignResponse.location:type_name -> master_pb.Location - 20, // 16: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection - 2, // 17: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage - 4, // 18: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage - 65, // 19: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry - 26, // 20: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo - 66, // 21: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry - 27, // 22: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo - 67, // 23: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry - 28, // 24: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo - 68, // 25: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry - 29, // 26: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo - 69, // 27: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation - 5, // 28: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend - 70, // 29: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode - 71, // 30: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers - 14, // 31: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location - 25, // 32: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 33: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 34: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 25, // 35: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo - 14, // 36: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location - 0, // 37: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat - 8, // 38: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest - 12, // 39: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest - 15, // 40: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest - 15, // 41: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest - 18, // 42: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest - 21, // 43: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest - 23, // 44: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest - 30, // 45: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest - 32, // 46: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest - 34, // 47: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest - 36, // 48: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest - 38, // 49: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest - 40, // 50: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest - 42, // 51: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest - 44, // 52: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest - 46, // 53: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest - 48, // 54: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest - 50, // 55: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest - 56, // 56: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest - 52, // 57: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest - 54, // 58: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest - 58, // 59: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest - 16, // 60: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest - 1, // 61: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse - 11, // 62: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse - 13, // 63: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse - 17, // 64: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse - 17, // 65: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse - 19, // 66: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse - 22, // 67: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse - 24, // 68: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse - 31, // 69: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse - 33, // 70: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse - 35, // 71: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse - 37, // 72: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse - 39, // 73: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse - 41, // 74: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse - 43, // 75: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse - 45, // 76: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse - 47, // 77: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse - 49, // 78: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse - 51, // 79: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse - 57, // 80: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse - 53, // 81: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse - 55, // 82: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse - 59, // 83: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse - 60, // 84: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse - 61, // [61:85] is the sub-list for method output_type - 37, // [37:61] is the sub-list for method input_type - 37, // [37:37] is the sub-list for extension type_name - 37, // [37:37] is the sub-list for extension extendee - 0, // [0:37] is the sub-list for field type_name + 70, // 6: master_pb.Heartbeat.max_volume_counts:type_name -> master_pb.Heartbeat.MaxVolumeCountsEntry + 81, // 7: master_pb.Heartbeat.state:type_name -> volume_server_pb.VolumeServerState + 61, // 8: master_pb.Heartbeat.block_volume_infos:type_name -> master_pb.BlockVolumeInfoMessage + 62, // 9: master_pb.Heartbeat.new_block_volumes:type_name -> master_pb.BlockVolumeShortInfoMessage + 62, // 10: master_pb.Heartbeat.deleted_block_volumes:type_name -> master_pb.BlockVolumeShortInfoMessage + 5, // 11: master_pb.HeartbeatResponse.storage_backends:type_name -> master_pb.StorageBackend + 63, // 12: master_pb.HeartbeatResponse.block_volume_assignments:type_name -> master_pb.BlockVolumeAssignment + 71, // 13: master_pb.StorageBackend.properties:type_name -> master_pb.StorageBackend.PropertiesEntry + 72, // 14: master_pb.SuperBlockExtra.erasure_coding:type_name -> master_pb.SuperBlockExtra.ErasureCoding + 9, // 15: master_pb.KeepConnectedResponse.volume_location:type_name -> master_pb.VolumeLocation + 10, // 16: master_pb.KeepConnectedResponse.cluster_node_update:type_name -> master_pb.ClusterNodeUpdate + 73, // 17: master_pb.LookupVolumeResponse.volume_id_locations:type_name -> master_pb.LookupVolumeResponse.VolumeIdLocation + 14, // 18: master_pb.AssignResponse.replicas:type_name -> master_pb.Location + 14, // 19: master_pb.AssignResponse.location:type_name -> master_pb.Location + 20, // 20: master_pb.CollectionListResponse.collections:type_name -> master_pb.Collection + 2, // 21: master_pb.DiskInfo.volume_infos:type_name -> master_pb.VolumeInformationMessage + 4, // 22: master_pb.DiskInfo.ec_shard_infos:type_name -> master_pb.VolumeEcShardInformationMessage + 74, // 23: master_pb.DataNodeInfo.diskInfos:type_name -> master_pb.DataNodeInfo.DiskInfosEntry + 26, // 24: master_pb.RackInfo.data_node_infos:type_name -> master_pb.DataNodeInfo + 75, // 25: master_pb.RackInfo.diskInfos:type_name -> master_pb.RackInfo.DiskInfosEntry + 27, // 26: master_pb.DataCenterInfo.rack_infos:type_name -> master_pb.RackInfo + 76, // 27: master_pb.DataCenterInfo.diskInfos:type_name -> master_pb.DataCenterInfo.DiskInfosEntry + 28, // 28: master_pb.TopologyInfo.data_center_infos:type_name -> master_pb.DataCenterInfo + 77, // 29: master_pb.TopologyInfo.diskInfos:type_name -> master_pb.TopologyInfo.DiskInfosEntry + 29, // 30: master_pb.VolumeListResponse.topology_info:type_name -> master_pb.TopologyInfo + 78, // 31: master_pb.LookupEcVolumeResponse.shard_id_locations:type_name -> master_pb.LookupEcVolumeResponse.EcShardIdLocation + 5, // 32: master_pb.GetMasterConfigurationResponse.storage_backends:type_name -> master_pb.StorageBackend + 79, // 33: master_pb.ListClusterNodesResponse.cluster_nodes:type_name -> master_pb.ListClusterNodesResponse.ClusterNode + 80, // 34: master_pb.RaftListClusterServersResponse.cluster_servers:type_name -> master_pb.RaftListClusterServersResponse.ClusterServers + 14, // 35: master_pb.LookupVolumeResponse.VolumeIdLocation.locations:type_name -> master_pb.Location + 25, // 36: master_pb.DataNodeInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 37: master_pb.RackInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 38: master_pb.DataCenterInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 25, // 39: master_pb.TopologyInfo.DiskInfosEntry.value:type_name -> master_pb.DiskInfo + 14, // 40: master_pb.LookupEcVolumeResponse.EcShardIdLocation.locations:type_name -> master_pb.Location + 0, // 41: master_pb.Seaweed.SendHeartbeat:input_type -> master_pb.Heartbeat + 8, // 42: master_pb.Seaweed.KeepConnected:input_type -> master_pb.KeepConnectedRequest + 12, // 43: master_pb.Seaweed.LookupVolume:input_type -> master_pb.LookupVolumeRequest + 15, // 44: master_pb.Seaweed.Assign:input_type -> master_pb.AssignRequest + 15, // 45: master_pb.Seaweed.StreamAssign:input_type -> master_pb.AssignRequest + 18, // 46: master_pb.Seaweed.Statistics:input_type -> master_pb.StatisticsRequest + 21, // 47: master_pb.Seaweed.CollectionList:input_type -> master_pb.CollectionListRequest + 23, // 48: master_pb.Seaweed.CollectionDelete:input_type -> master_pb.CollectionDeleteRequest + 30, // 49: master_pb.Seaweed.VolumeList:input_type -> master_pb.VolumeListRequest + 32, // 50: master_pb.Seaweed.LookupEcVolume:input_type -> master_pb.LookupEcVolumeRequest + 34, // 51: master_pb.Seaweed.VacuumVolume:input_type -> master_pb.VacuumVolumeRequest + 36, // 52: master_pb.Seaweed.DisableVacuum:input_type -> master_pb.DisableVacuumRequest + 38, // 53: master_pb.Seaweed.EnableVacuum:input_type -> master_pb.EnableVacuumRequest + 40, // 54: master_pb.Seaweed.VolumeMarkReadonly:input_type -> master_pb.VolumeMarkReadonlyRequest + 42, // 55: master_pb.Seaweed.GetMasterConfiguration:input_type -> master_pb.GetMasterConfigurationRequest + 44, // 56: master_pb.Seaweed.ListClusterNodes:input_type -> master_pb.ListClusterNodesRequest + 46, // 57: master_pb.Seaweed.LeaseAdminToken:input_type -> master_pb.LeaseAdminTokenRequest + 48, // 58: master_pb.Seaweed.ReleaseAdminToken:input_type -> master_pb.ReleaseAdminTokenRequest + 50, // 59: master_pb.Seaweed.Ping:input_type -> master_pb.PingRequest + 56, // 60: master_pb.Seaweed.RaftListClusterServers:input_type -> master_pb.RaftListClusterServersRequest + 52, // 61: master_pb.Seaweed.RaftAddServer:input_type -> master_pb.RaftAddServerRequest + 54, // 62: master_pb.Seaweed.RaftRemoveServer:input_type -> master_pb.RaftRemoveServerRequest + 58, // 63: master_pb.Seaweed.RaftLeadershipTransfer:input_type -> master_pb.RaftLeadershipTransferRequest + 16, // 64: master_pb.Seaweed.VolumeGrow:input_type -> master_pb.VolumeGrowRequest + 64, // 65: master_pb.Seaweed.CreateBlockVolume:input_type -> master_pb.CreateBlockVolumeRequest + 66, // 66: master_pb.Seaweed.DeleteBlockVolume:input_type -> master_pb.DeleteBlockVolumeRequest + 68, // 67: master_pb.Seaweed.LookupBlockVolume:input_type -> master_pb.LookupBlockVolumeRequest + 1, // 68: master_pb.Seaweed.SendHeartbeat:output_type -> master_pb.HeartbeatResponse + 11, // 69: master_pb.Seaweed.KeepConnected:output_type -> master_pb.KeepConnectedResponse + 13, // 70: master_pb.Seaweed.LookupVolume:output_type -> master_pb.LookupVolumeResponse + 17, // 71: master_pb.Seaweed.Assign:output_type -> master_pb.AssignResponse + 17, // 72: master_pb.Seaweed.StreamAssign:output_type -> master_pb.AssignResponse + 19, // 73: master_pb.Seaweed.Statistics:output_type -> master_pb.StatisticsResponse + 22, // 74: master_pb.Seaweed.CollectionList:output_type -> master_pb.CollectionListResponse + 24, // 75: master_pb.Seaweed.CollectionDelete:output_type -> master_pb.CollectionDeleteResponse + 31, // 76: master_pb.Seaweed.VolumeList:output_type -> master_pb.VolumeListResponse + 33, // 77: master_pb.Seaweed.LookupEcVolume:output_type -> master_pb.LookupEcVolumeResponse + 35, // 78: master_pb.Seaweed.VacuumVolume:output_type -> master_pb.VacuumVolumeResponse + 37, // 79: master_pb.Seaweed.DisableVacuum:output_type -> master_pb.DisableVacuumResponse + 39, // 80: master_pb.Seaweed.EnableVacuum:output_type -> master_pb.EnableVacuumResponse + 41, // 81: master_pb.Seaweed.VolumeMarkReadonly:output_type -> master_pb.VolumeMarkReadonlyResponse + 43, // 82: master_pb.Seaweed.GetMasterConfiguration:output_type -> master_pb.GetMasterConfigurationResponse + 45, // 83: master_pb.Seaweed.ListClusterNodes:output_type -> master_pb.ListClusterNodesResponse + 47, // 84: master_pb.Seaweed.LeaseAdminToken:output_type -> master_pb.LeaseAdminTokenResponse + 49, // 85: master_pb.Seaweed.ReleaseAdminToken:output_type -> master_pb.ReleaseAdminTokenResponse + 51, // 86: master_pb.Seaweed.Ping:output_type -> master_pb.PingResponse + 57, // 87: master_pb.Seaweed.RaftListClusterServers:output_type -> master_pb.RaftListClusterServersResponse + 53, // 88: master_pb.Seaweed.RaftAddServer:output_type -> master_pb.RaftAddServerResponse + 55, // 89: master_pb.Seaweed.RaftRemoveServer:output_type -> master_pb.RaftRemoveServerResponse + 59, // 90: master_pb.Seaweed.RaftLeadershipTransfer:output_type -> master_pb.RaftLeadershipTransferResponse + 60, // 91: master_pb.Seaweed.VolumeGrow:output_type -> master_pb.VolumeGrowResponse + 65, // 92: master_pb.Seaweed.CreateBlockVolume:output_type -> master_pb.CreateBlockVolumeResponse + 67, // 93: master_pb.Seaweed.DeleteBlockVolume:output_type -> master_pb.DeleteBlockVolumeResponse + 69, // 94: master_pb.Seaweed.LookupBlockVolume:output_type -> master_pb.LookupBlockVolumeResponse + 68, // [68:95] is the sub-list for method output_type + 41, // [41:68] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_master_proto_init() } @@ -4750,7 +5442,7 @@ func file_master_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_master_proto_rawDesc), len(file_master_proto_rawDesc)), NumEnums: 0, - NumMessages: 72, + NumMessages: 81, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/master_pb/master_grpc.pb.go b/weed/pb/master_pb/master_grpc.pb.go index f13d53bca..ad2359292 100644 --- a/weed/pb/master_pb/master_grpc.pb.go +++ b/weed/pb/master_pb/master_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.4 +// - protoc-gen-go-grpc v1.6.1 +// - protoc v5.29.3 // source: master.proto package master_pb @@ -43,6 +43,9 @@ const ( Seaweed_RaftRemoveServer_FullMethodName = "/master_pb.Seaweed/RaftRemoveServer" Seaweed_RaftLeadershipTransfer_FullMethodName = "/master_pb.Seaweed/RaftLeadershipTransfer" Seaweed_VolumeGrow_FullMethodName = "/master_pb.Seaweed/VolumeGrow" + Seaweed_CreateBlockVolume_FullMethodName = "/master_pb.Seaweed/CreateBlockVolume" + Seaweed_DeleteBlockVolume_FullMethodName = "/master_pb.Seaweed/DeleteBlockVolume" + Seaweed_LookupBlockVolume_FullMethodName = "/master_pb.Seaweed/LookupBlockVolume" ) // SeaweedClient is the client API for Seaweed service. @@ -73,6 +76,9 @@ type SeaweedClient interface { RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) RaftLeadershipTransfer(ctx context.Context, in *RaftLeadershipTransferRequest, opts ...grpc.CallOption) (*RaftLeadershipTransferResponse, error) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, opts ...grpc.CallOption) (*VolumeGrowResponse, error) + CreateBlockVolume(ctx context.Context, in *CreateBlockVolumeRequest, opts ...grpc.CallOption) (*CreateBlockVolumeResponse, error) + DeleteBlockVolume(ctx context.Context, in *DeleteBlockVolumeRequest, opts ...grpc.CallOption) (*DeleteBlockVolumeResponse, error) + LookupBlockVolume(ctx context.Context, in *LookupBlockVolumeRequest, opts ...grpc.CallOption) (*LookupBlockVolumeResponse, error) } type seaweedClient struct { @@ -332,6 +338,36 @@ func (c *seaweedClient) VolumeGrow(ctx context.Context, in *VolumeGrowRequest, o return out, nil } +func (c *seaweedClient) CreateBlockVolume(ctx context.Context, in *CreateBlockVolumeRequest, opts ...grpc.CallOption) (*CreateBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateBlockVolumeResponse) + err := c.cc.Invoke(ctx, Seaweed_CreateBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) DeleteBlockVolume(ctx context.Context, in *DeleteBlockVolumeRequest, opts ...grpc.CallOption) (*DeleteBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteBlockVolumeResponse) + err := c.cc.Invoke(ctx, Seaweed_DeleteBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedClient) LookupBlockVolume(ctx context.Context, in *LookupBlockVolumeRequest, opts ...grpc.CallOption) (*LookupBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupBlockVolumeResponse) + err := c.cc.Invoke(ctx, Seaweed_LookupBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // SeaweedServer is the server API for Seaweed service. // All implementations must embed UnimplementedSeaweedServer // for forward compatibility. @@ -360,6 +396,9 @@ type SeaweedServer interface { RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) + CreateBlockVolume(context.Context, *CreateBlockVolumeRequest) (*CreateBlockVolumeResponse, error) + DeleteBlockVolume(context.Context, *DeleteBlockVolumeRequest) (*DeleteBlockVolumeResponse, error) + LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error) mustEmbedUnimplementedSeaweedServer() } @@ -371,76 +410,85 @@ type SeaweedServer interface { type UnimplementedSeaweedServer struct{} func (UnimplementedSeaweedServer) SendHeartbeat(grpc.BidiStreamingServer[Heartbeat, HeartbeatResponse]) error { - return status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") + return status.Error(codes.Unimplemented, "method SendHeartbeat not implemented") } func (UnimplementedSeaweedServer) KeepConnected(grpc.BidiStreamingServer[KeepConnectedRequest, KeepConnectedResponse]) error { - return status.Errorf(codes.Unimplemented, "method KeepConnected not implemented") + return status.Error(codes.Unimplemented, "method KeepConnected not implemented") } func (UnimplementedSeaweedServer) LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method LookupVolume not implemented") } func (UnimplementedSeaweedServer) Assign(context.Context, *AssignRequest) (*AssignResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Assign not implemented") + return nil, status.Error(codes.Unimplemented, "method Assign not implemented") } func (UnimplementedSeaweedServer) StreamAssign(grpc.BidiStreamingServer[AssignRequest, AssignResponse]) error { - return status.Errorf(codes.Unimplemented, "method StreamAssign not implemented") + return status.Error(codes.Unimplemented, "method StreamAssign not implemented") } func (UnimplementedSeaweedServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented") + return nil, status.Error(codes.Unimplemented, "method Statistics not implemented") } func (UnimplementedSeaweedServer) CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CollectionList not implemented") + return nil, status.Error(codes.Unimplemented, "method CollectionList not implemented") } func (UnimplementedSeaweedServer) CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CollectionDelete not implemented") + return nil, status.Error(codes.Unimplemented, "method CollectionDelete not implemented") } func (UnimplementedSeaweedServer) VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeList not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeList not implemented") } func (UnimplementedSeaweedServer) LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LookupEcVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method LookupEcVolume not implemented") } func (UnimplementedSeaweedServer) VacuumVolume(context.Context, *VacuumVolumeRequest) (*VacuumVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method VacuumVolume not implemented") } func (UnimplementedSeaweedServer) DisableVacuum(context.Context, *DisableVacuumRequest) (*DisableVacuumResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DisableVacuum not implemented") + return nil, status.Error(codes.Unimplemented, "method DisableVacuum not implemented") } func (UnimplementedSeaweedServer) EnableVacuum(context.Context, *EnableVacuumRequest) (*EnableVacuumResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method EnableVacuum not implemented") + return nil, status.Error(codes.Unimplemented, "method EnableVacuum not implemented") } func (UnimplementedSeaweedServer) VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkReadonly not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeMarkReadonly not implemented") } func (UnimplementedSeaweedServer) GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetMasterConfiguration not implemented") + return nil, status.Error(codes.Unimplemented, "method GetMasterConfiguration not implemented") } func (UnimplementedSeaweedServer) ListClusterNodes(context.Context, *ListClusterNodesRequest) (*ListClusterNodesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ListClusterNodes not implemented") + return nil, status.Error(codes.Unimplemented, "method ListClusterNodes not implemented") } func (UnimplementedSeaweedServer) LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method LeaseAdminToken not implemented") + return nil, status.Error(codes.Unimplemented, "method LeaseAdminToken not implemented") } func (UnimplementedSeaweedServer) ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReleaseAdminToken not implemented") + return nil, status.Error(codes.Unimplemented, "method ReleaseAdminToken not implemented") } func (UnimplementedSeaweedServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") + return nil, status.Error(codes.Unimplemented, "method Ping not implemented") } func (UnimplementedSeaweedServer) RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RaftListClusterServers not implemented") + return nil, status.Error(codes.Unimplemented, "method RaftListClusterServers not implemented") } func (UnimplementedSeaweedServer) RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RaftAddServer not implemented") + return nil, status.Error(codes.Unimplemented, "method RaftAddServer not implemented") } func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RaftRemoveServer not implemented") + return nil, status.Error(codes.Unimplemented, "method RaftRemoveServer not implemented") } func (UnimplementedSeaweedServer) RaftLeadershipTransfer(context.Context, *RaftLeadershipTransferRequest) (*RaftLeadershipTransferResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RaftLeadershipTransfer not implemented") + return nil, status.Error(codes.Unimplemented, "method RaftLeadershipTransfer not implemented") } func (UnimplementedSeaweedServer) VolumeGrow(context.Context, *VolumeGrowRequest) (*VolumeGrowResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeGrow not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeGrow not implemented") +} +func (UnimplementedSeaweedServer) CreateBlockVolume(context.Context, *CreateBlockVolumeRequest) (*CreateBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CreateBlockVolume not implemented") +} +func (UnimplementedSeaweedServer) DeleteBlockVolume(context.Context, *DeleteBlockVolumeRequest) (*DeleteBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteBlockVolume not implemented") +} +func (UnimplementedSeaweedServer) LookupBlockVolume(context.Context, *LookupBlockVolumeRequest) (*LookupBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method LookupBlockVolume not implemented") } func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {} func (UnimplementedSeaweedServer) testEmbeddedByValue() {} @@ -453,7 +501,7 @@ type UnsafeSeaweedServer interface { } func RegisterSeaweedServer(s grpc.ServiceRegistrar, srv SeaweedServer) { - // If the following call pancis, it indicates UnimplementedSeaweedServer was + // If the following call panics, it indicates UnimplementedSeaweedServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. @@ -862,6 +910,60 @@ func _Seaweed_VolumeGrow_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _Seaweed_CreateBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).CreateBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_CreateBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).CreateBlockVolume(ctx, req.(*CreateBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_DeleteBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).DeleteBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_DeleteBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).DeleteBlockVolume(ctx, req.(*DeleteBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Seaweed_LookupBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedServer).LookupBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Seaweed_LookupBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedServer).LookupBlockVolume(ctx, req.(*LookupBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -953,6 +1055,18 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{ MethodName: "VolumeGrow", Handler: _Seaweed_VolumeGrow_Handler, }, + { + MethodName: "CreateBlockVolume", + Handler: _Seaweed_CreateBlockVolume_Handler, + }, + { + MethodName: "DeleteBlockVolume", + Handler: _Seaweed_DeleteBlockVolume_Handler, + }, + { + MethodName: "LookupBlockVolume", + Handler: _Seaweed_LookupBlockVolume_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index c1a9282bd..e7f675e6a 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -138,6 +138,12 @@ service VolumeServer { rpc Ping (PingRequest) returns (PingResponse) { } + // block volumes + rpc AllocateBlockVolume (AllocateBlockVolumeRequest) returns (AllocateBlockVolumeResponse) { + } + rpc VolumeServerDeleteBlockVolume (VolumeServerDeleteBlockVolumeRequest) returns (VolumeServerDeleteBlockVolumeResponse) { + } + } ////////////////////////////////////////////////// @@ -756,3 +762,24 @@ message PingResponse { int64 remote_time_ns = 2; int64 stop_time_ns = 3; } + +////////////////////////////////////////////////// +// Block volume messages (volume_server_pb package) +////////////////////////////////////////////////// + +message AllocateBlockVolumeRequest { + string name = 1; + uint64 size_bytes = 2; + string disk_type = 3; +} +message AllocateBlockVolumeResponse { + string path = 1; + string iqn = 2; + string iscsi_addr = 3; +} + +message VolumeServerDeleteBlockVolumeRequest { + string name = 1; +} +message VolumeServerDeleteBlockVolumeResponse { +} diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 63c8cbc51..018f1b0f2 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 -// protoc v6.33.4 +// protoc-gen-go v1.36.11 +// protoc v5.29.3 // source: volume_server.proto package volume_server_pb @@ -6185,6 +6185,206 @@ func (x *PingResponse) GetStopTimeNs() int64 { return 0 } +type AllocateBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + DiskType string `protobuf:"bytes,3,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AllocateBlockVolumeRequest) Reset() { + *x = AllocateBlockVolumeRequest{} + mi := &file_volume_server_proto_msgTypes[107] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AllocateBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllocateBlockVolumeRequest) ProtoMessage() {} + +func (x *AllocateBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[107] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllocateBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*AllocateBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{107} +} + +func (x *AllocateBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *AllocateBlockVolumeRequest) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *AllocateBlockVolumeRequest) GetDiskType() string { + if x != nil { + return x.DiskType + } + return "" +} + +type AllocateBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Iqn string `protobuf:"bytes,2,opt,name=iqn,proto3" json:"iqn,omitempty"` + IscsiAddr string `protobuf:"bytes,3,opt,name=iscsi_addr,json=iscsiAddr,proto3" json:"iscsi_addr,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AllocateBlockVolumeResponse) Reset() { + *x = AllocateBlockVolumeResponse{} + mi := &file_volume_server_proto_msgTypes[108] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AllocateBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllocateBlockVolumeResponse) ProtoMessage() {} + +func (x *AllocateBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[108] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllocateBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*AllocateBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{108} +} + +func (x *AllocateBlockVolumeResponse) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *AllocateBlockVolumeResponse) GetIqn() string { + if x != nil { + return x.Iqn + } + return "" +} + +func (x *AllocateBlockVolumeResponse) GetIscsiAddr() string { + if x != nil { + return x.IscsiAddr + } + return "" +} + +type VolumeServerDeleteBlockVolumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeServerDeleteBlockVolumeRequest) Reset() { + *x = VolumeServerDeleteBlockVolumeRequest{} + mi := &file_volume_server_proto_msgTypes[109] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeServerDeleteBlockVolumeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeServerDeleteBlockVolumeRequest) ProtoMessage() {} + +func (x *VolumeServerDeleteBlockVolumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[109] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeServerDeleteBlockVolumeRequest.ProtoReflect.Descriptor instead. +func (*VolumeServerDeleteBlockVolumeRequest) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{109} +} + +func (x *VolumeServerDeleteBlockVolumeRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type VolumeServerDeleteBlockVolumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeServerDeleteBlockVolumeResponse) Reset() { + *x = VolumeServerDeleteBlockVolumeResponse{} + mi := &file_volume_server_proto_msgTypes[110] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeServerDeleteBlockVolumeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeServerDeleteBlockVolumeResponse) ProtoMessage() {} + +func (x *VolumeServerDeleteBlockVolumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_volume_server_proto_msgTypes[110] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeServerDeleteBlockVolumeResponse.ProtoReflect.Descriptor instead. +func (*VolumeServerDeleteBlockVolumeResponse) Descriptor() ([]byte, []int) { + return file_volume_server_proto_rawDescGZIP(), []int{110} +} + type FetchAndWriteNeedleRequest_Replica struct { state protoimpl.MessageState `protogen:"open.v1"` Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` @@ -6196,7 +6396,7 @@ type FetchAndWriteNeedleRequest_Replica struct { func (x *FetchAndWriteNeedleRequest_Replica) Reset() { *x = FetchAndWriteNeedleRequest_Replica{} - mi := &file_volume_server_proto_msgTypes[107] + mi := &file_volume_server_proto_msgTypes[111] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6208,7 +6408,7 @@ func (x *FetchAndWriteNeedleRequest_Replica) String() string { func (*FetchAndWriteNeedleRequest_Replica) ProtoMessage() {} func (x *FetchAndWriteNeedleRequest_Replica) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[107] + mi := &file_volume_server_proto_msgTypes[111] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6256,7 +6456,7 @@ type QueryRequest_Filter struct { func (x *QueryRequest_Filter) Reset() { *x = QueryRequest_Filter{} - mi := &file_volume_server_proto_msgTypes[108] + mi := &file_volume_server_proto_msgTypes[112] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6268,7 +6468,7 @@ func (x *QueryRequest_Filter) String() string { func (*QueryRequest_Filter) ProtoMessage() {} func (x *QueryRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[108] + mi := &file_volume_server_proto_msgTypes[112] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6318,7 +6518,7 @@ type QueryRequest_InputSerialization struct { func (x *QueryRequest_InputSerialization) Reset() { *x = QueryRequest_InputSerialization{} - mi := &file_volume_server_proto_msgTypes[109] + mi := &file_volume_server_proto_msgTypes[113] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6330,7 +6530,7 @@ func (x *QueryRequest_InputSerialization) String() string { func (*QueryRequest_InputSerialization) ProtoMessage() {} func (x *QueryRequest_InputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[109] + mi := &file_volume_server_proto_msgTypes[113] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6384,7 +6584,7 @@ type QueryRequest_OutputSerialization struct { func (x *QueryRequest_OutputSerialization) Reset() { *x = QueryRequest_OutputSerialization{} - mi := &file_volume_server_proto_msgTypes[110] + mi := &file_volume_server_proto_msgTypes[114] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6396,7 +6596,7 @@ func (x *QueryRequest_OutputSerialization) String() string { func (*QueryRequest_OutputSerialization) ProtoMessage() {} func (x *QueryRequest_OutputSerialization) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[110] + mi := &file_volume_server_proto_msgTypes[114] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6442,7 +6642,7 @@ type QueryRequest_InputSerialization_CSVInput struct { func (x *QueryRequest_InputSerialization_CSVInput) Reset() { *x = QueryRequest_InputSerialization_CSVInput{} - mi := &file_volume_server_proto_msgTypes[111] + mi := &file_volume_server_proto_msgTypes[115] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6454,7 +6654,7 @@ func (x *QueryRequest_InputSerialization_CSVInput) String() string { func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_CSVInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[111] + mi := &file_volume_server_proto_msgTypes[115] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6528,7 +6728,7 @@ type QueryRequest_InputSerialization_JSONInput struct { func (x *QueryRequest_InputSerialization_JSONInput) Reset() { *x = QueryRequest_InputSerialization_JSONInput{} - mi := &file_volume_server_proto_msgTypes[112] + mi := &file_volume_server_proto_msgTypes[116] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6540,7 +6740,7 @@ func (x *QueryRequest_InputSerialization_JSONInput) String() string { func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_JSONInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[112] + mi := &file_volume_server_proto_msgTypes[116] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6571,7 +6771,7 @@ type QueryRequest_InputSerialization_ParquetInput struct { func (x *QueryRequest_InputSerialization_ParquetInput) Reset() { *x = QueryRequest_InputSerialization_ParquetInput{} - mi := &file_volume_server_proto_msgTypes[113] + mi := &file_volume_server_proto_msgTypes[117] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6583,7 +6783,7 @@ func (x *QueryRequest_InputSerialization_ParquetInput) String() string { func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {} func (x *QueryRequest_InputSerialization_ParquetInput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[113] + mi := &file_volume_server_proto_msgTypes[117] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6612,7 +6812,7 @@ type QueryRequest_OutputSerialization_CSVOutput struct { func (x *QueryRequest_OutputSerialization_CSVOutput) Reset() { *x = QueryRequest_OutputSerialization_CSVOutput{} - mi := &file_volume_server_proto_msgTypes[114] + mi := &file_volume_server_proto_msgTypes[118] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6624,7 +6824,7 @@ func (x *QueryRequest_OutputSerialization_CSVOutput) String() string { func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_CSVOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[114] + mi := &file_volume_server_proto_msgTypes[118] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6684,7 +6884,7 @@ type QueryRequest_OutputSerialization_JSONOutput struct { func (x *QueryRequest_OutputSerialization_JSONOutput) Reset() { *x = QueryRequest_OutputSerialization_JSONOutput{} - mi := &file_volume_server_proto_msgTypes[115] + mi := &file_volume_server_proto_msgTypes[119] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6696,7 +6896,7 @@ func (x *QueryRequest_OutputSerialization_JSONOutput) String() string { func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {} func (x *QueryRequest_OutputSerialization_JSONOutput) ProtoReflect() protoreflect.Message { - mi := &file_volume_server_proto_msgTypes[115] + mi := &file_volume_server_proto_msgTypes[119] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7222,12 +7422,25 @@ const file_volume_server_proto_rawDesc = "" + "\rstart_time_ns\x18\x01 \x01(\x03R\vstartTimeNs\x12$\n" + "\x0eremote_time_ns\x18\x02 \x01(\x03R\fremoteTimeNs\x12 \n" + "\fstop_time_ns\x18\x03 \x01(\x03R\n" + - "stopTimeNs*>\n" + + "stopTimeNs\"l\n" + + "\x1aAllocateBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\n" + + "size_bytes\x18\x02 \x01(\x04R\tsizeBytes\x12\x1b\n" + + "\tdisk_type\x18\x03 \x01(\tR\bdiskType\"b\n" + + "\x1bAllocateBlockVolumeResponse\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" + + "\x03iqn\x18\x02 \x01(\tR\x03iqn\x12\x1d\n" + + "\n" + + "iscsi_addr\x18\x03 \x01(\tR\tiscsiAddr\":\n" + + "$VolumeServerDeleteBlockVolumeRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"'\n" + + "%VolumeServerDeleteBlockVolumeResponse*>\n" + "\x0fVolumeScrubMode\x12\v\n" + "\aUNKNOWN\x10\x00\x12\t\n" + "\x05INDEX\x10\x01\x12\b\n" + "\x04FULL\x10\x02\x12\t\n" + - "\x05LOCAL\x10\x032\xfb(\n" + + "\x05LOCAL\x10\x032\x86+\n" + "\fVolumeServer\x12\\\n" + "\vBatchDelete\x12$.volume_server_pb.BatchDeleteRequest\x1a%.volume_server_pb.BatchDeleteResponse\"\x00\x12n\n" + "\x11VacuumVolumeCheck\x12*.volume_server_pb.VacuumVolumeCheckRequest\x1a+.volume_server_pb.VacuumVolumeCheckResponse\"\x00\x12v\n" + @@ -7277,7 +7490,9 @@ const file_volume_server_proto_rawDesc = "" + "\rScrubEcVolume\x12&.volume_server_pb.ScrubEcVolumeRequest\x1a'.volume_server_pb.ScrubEcVolumeResponse\"\x00\x12L\n" + "\x05Query\x12\x1e.volume_server_pb.QueryRequest\x1a\x1f.volume_server_pb.QueriedStripe\"\x000\x01\x12q\n" + "\x12VolumeNeedleStatus\x12+.volume_server_pb.VolumeNeedleStatusRequest\x1a,.volume_server_pb.VolumeNeedleStatusResponse\"\x00\x12G\n" + - "\x04Ping\x12\x1d.volume_server_pb.PingRequest\x1a\x1e.volume_server_pb.PingResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3" + "\x04Ping\x12\x1d.volume_server_pb.PingRequest\x1a\x1e.volume_server_pb.PingResponse\"\x00\x12t\n" + + "\x13AllocateBlockVolume\x12,.volume_server_pb.AllocateBlockVolumeRequest\x1a-.volume_server_pb.AllocateBlockVolumeResponse\"\x00\x12\x92\x01\n" + + "\x1dVolumeServerDeleteBlockVolume\x126.volume_server_pb.VolumeServerDeleteBlockVolumeRequest\x1a7.volume_server_pb.VolumeServerDeleteBlockVolumeResponse\"\x00B9Z7github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pbb\x06proto3" var ( file_volume_server_proto_rawDescOnce sync.Once @@ -7292,7 +7507,7 @@ func file_volume_server_proto_rawDescGZIP() []byte { } var file_volume_server_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 116) +var file_volume_server_proto_msgTypes = make([]protoimpl.MessageInfo, 120) var file_volume_server_proto_goTypes = []any{ (VolumeScrubMode)(0), // 0: volume_server_pb.VolumeScrubMode (*VolumeServerState)(nil), // 1: volume_server_pb.VolumeServerState @@ -7402,17 +7617,21 @@ var file_volume_server_proto_goTypes = []any{ (*VolumeNeedleStatusResponse)(nil), // 105: volume_server_pb.VolumeNeedleStatusResponse (*PingRequest)(nil), // 106: volume_server_pb.PingRequest (*PingResponse)(nil), // 107: volume_server_pb.PingResponse - (*FetchAndWriteNeedleRequest_Replica)(nil), // 108: volume_server_pb.FetchAndWriteNeedleRequest.Replica - (*QueryRequest_Filter)(nil), // 109: volume_server_pb.QueryRequest.Filter - (*QueryRequest_InputSerialization)(nil), // 110: volume_server_pb.QueryRequest.InputSerialization - (*QueryRequest_OutputSerialization)(nil), // 111: volume_server_pb.QueryRequest.OutputSerialization - (*QueryRequest_InputSerialization_CSVInput)(nil), // 112: volume_server_pb.QueryRequest.InputSerialization.CSVInput - (*QueryRequest_InputSerialization_JSONInput)(nil), // 113: volume_server_pb.QueryRequest.InputSerialization.JSONInput - (*QueryRequest_InputSerialization_ParquetInput)(nil), // 114: volume_server_pb.QueryRequest.InputSerialization.ParquetInput - (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 115: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 116: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput - (*remote_pb.RemoteConf)(nil), // 117: remote_pb.RemoteConf - (*remote_pb.RemoteStorageLocation)(nil), // 118: remote_pb.RemoteStorageLocation + (*AllocateBlockVolumeRequest)(nil), // 108: volume_server_pb.AllocateBlockVolumeRequest + (*AllocateBlockVolumeResponse)(nil), // 109: volume_server_pb.AllocateBlockVolumeResponse + (*VolumeServerDeleteBlockVolumeRequest)(nil), // 110: volume_server_pb.VolumeServerDeleteBlockVolumeRequest + (*VolumeServerDeleteBlockVolumeResponse)(nil), // 111: volume_server_pb.VolumeServerDeleteBlockVolumeResponse + (*FetchAndWriteNeedleRequest_Replica)(nil), // 112: volume_server_pb.FetchAndWriteNeedleRequest.Replica + (*QueryRequest_Filter)(nil), // 113: volume_server_pb.QueryRequest.Filter + (*QueryRequest_InputSerialization)(nil), // 114: volume_server_pb.QueryRequest.InputSerialization + (*QueryRequest_OutputSerialization)(nil), // 115: volume_server_pb.QueryRequest.OutputSerialization + (*QueryRequest_InputSerialization_CSVInput)(nil), // 116: volume_server_pb.QueryRequest.InputSerialization.CSVInput + (*QueryRequest_InputSerialization_JSONInput)(nil), // 117: volume_server_pb.QueryRequest.InputSerialization.JSONInput + (*QueryRequest_InputSerialization_ParquetInput)(nil), // 118: volume_server_pb.QueryRequest.InputSerialization.ParquetInput + (*QueryRequest_OutputSerialization_CSVOutput)(nil), // 119: volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + (*QueryRequest_OutputSerialization_JSONOutput)(nil), // 120: volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + (*remote_pb.RemoteConf)(nil), // 121: remote_pb.RemoteConf + (*remote_pb.RemoteStorageLocation)(nil), // 122: remote_pb.RemoteStorageLocation } var file_volume_server_proto_depIdxs = []int32{ 4, // 0: volume_server_pb.BatchDeleteResponse.results:type_name -> volume_server_pb.DeleteResult @@ -7428,20 +7647,20 @@ var file_volume_server_proto_depIdxs = []int32{ 82, // 10: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus 83, // 11: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus 1, // 12: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState - 108, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica - 117, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf - 118, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation + 112, // 13: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica + 121, // 14: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf + 122, // 15: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation 0, // 16: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode 0, // 17: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode 79, // 18: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo - 109, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter - 110, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization - 111, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization - 112, // 22: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput - 113, // 23: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput - 114, // 24: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput - 115, // 25: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput - 116, // 26: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput + 113, // 19: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter + 114, // 20: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization + 115, // 21: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization + 116, // 22: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput + 117, // 23: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput + 118, // 24: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput + 119, // 25: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput + 120, // 26: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput 2, // 27: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest 6, // 28: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest 8, // 29: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest @@ -7490,56 +7709,60 @@ var file_volume_server_proto_depIdxs = []int32{ 102, // 72: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest 104, // 73: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest 106, // 74: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest - 3, // 75: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse - 7, // 76: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse - 9, // 77: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse - 11, // 78: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse - 13, // 79: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse - 15, // 80: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse - 17, // 81: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse - 19, // 82: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse - 21, // 83: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse - 23, // 84: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse - 25, // 85: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse - 27, // 86: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse - 29, // 87: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse - 31, // 88: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse - 33, // 89: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse - 35, // 90: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse - 37, // 91: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse - 39, // 92: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse - 41, // 93: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse - 81, // 94: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse - 43, // 95: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse - 46, // 96: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse - 48, // 97: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse - 50, // 98: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse - 52, // 99: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse - 54, // 100: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse - 56, // 101: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse - 58, // 102: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse - 60, // 103: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse - 62, // 104: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse - 64, // 105: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse - 66, // 106: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse - 68, // 107: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse - 70, // 108: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse - 72, // 109: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse - 74, // 110: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse - 76, // 111: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse - 78, // 112: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse - 89, // 113: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse - 91, // 114: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse - 93, // 115: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse - 95, // 116: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse - 97, // 117: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse - 99, // 118: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse - 101, // 119: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse - 103, // 120: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe - 105, // 121: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse - 107, // 122: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse - 75, // [75:123] is the sub-list for method output_type - 27, // [27:75] is the sub-list for method input_type + 108, // 75: volume_server_pb.VolumeServer.AllocateBlockVolume:input_type -> volume_server_pb.AllocateBlockVolumeRequest + 110, // 76: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:input_type -> volume_server_pb.VolumeServerDeleteBlockVolumeRequest + 3, // 77: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse + 7, // 78: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse + 9, // 79: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse + 11, // 80: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse + 13, // 81: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse + 15, // 82: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse + 17, // 83: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse + 19, // 84: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse + 21, // 85: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse + 23, // 86: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse + 25, // 87: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse + 27, // 88: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse + 29, // 89: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse + 31, // 90: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse + 33, // 91: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse + 35, // 92: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse + 37, // 93: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse + 39, // 94: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse + 41, // 95: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse + 81, // 96: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse + 43, // 97: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse + 46, // 98: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse + 48, // 99: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse + 50, // 100: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse + 52, // 101: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse + 54, // 102: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse + 56, // 103: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse + 58, // 104: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse + 60, // 105: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse + 62, // 106: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse + 64, // 107: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse + 66, // 108: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse + 68, // 109: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse + 70, // 110: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse + 72, // 111: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse + 74, // 112: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse + 76, // 113: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse + 78, // 114: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse + 89, // 115: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse + 91, // 116: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse + 93, // 117: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse + 95, // 118: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse + 97, // 119: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse + 99, // 120: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse + 101, // 121: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse + 103, // 122: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe + 105, // 123: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse + 107, // 124: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse + 109, // 125: volume_server_pb.VolumeServer.AllocateBlockVolume:output_type -> volume_server_pb.AllocateBlockVolumeResponse + 111, // 126: volume_server_pb.VolumeServer.VolumeServerDeleteBlockVolume:output_type -> volume_server_pb.VolumeServerDeleteBlockVolumeResponse + 77, // [77:127] is the sub-list for method output_type + 27, // [27:77] is the sub-list for method input_type 27, // [27:27] is the sub-list for extension type_name 27, // [27:27] is the sub-list for extension extendee 0, // [0:27] is the sub-list for field type_name @@ -7560,7 +7783,7 @@ func file_volume_server_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_volume_server_proto_rawDesc), len(file_volume_server_proto_rawDesc)), NumEnums: 1, - NumMessages: 116, + NumMessages: 120, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/volume_server_pb/volume_server_grpc.pb.go b/weed/pb/volume_server_pb/volume_server_grpc.pb.go index 1a7c4905a..d4bb48802 100644 --- a/weed/pb/volume_server_pb/volume_server_grpc.pb.go +++ b/weed/pb/volume_server_pb/volume_server_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 -// - protoc v6.33.4 +// - protoc-gen-go-grpc v1.6.1 +// - protoc v5.29.3 // source: volume_server.proto package volume_server_pb @@ -19,54 +19,56 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - VolumeServer_BatchDelete_FullMethodName = "/volume_server_pb.VolumeServer/BatchDelete" - VolumeServer_VacuumVolumeCheck_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCheck" - VolumeServer_VacuumVolumeCompact_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCompact" - VolumeServer_VacuumVolumeCommit_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCommit" - VolumeServer_VacuumVolumeCleanup_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCleanup" - VolumeServer_DeleteCollection_FullMethodName = "/volume_server_pb.VolumeServer/DeleteCollection" - VolumeServer_AllocateVolume_FullMethodName = "/volume_server_pb.VolumeServer/AllocateVolume" - VolumeServer_VolumeSyncStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeSyncStatus" - VolumeServer_VolumeIncrementalCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeIncrementalCopy" - VolumeServer_VolumeMount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMount" - VolumeServer_VolumeUnmount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeUnmount" - VolumeServer_VolumeDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeDelete" - VolumeServer_VolumeMarkReadonly_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMarkReadonly" - VolumeServer_VolumeMarkWritable_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMarkWritable" - VolumeServer_VolumeConfigure_FullMethodName = "/volume_server_pb.VolumeServer/VolumeConfigure" - VolumeServer_VolumeStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeStatus" - VolumeServer_GetState_FullMethodName = "/volume_server_pb.VolumeServer/GetState" - VolumeServer_SetState_FullMethodName = "/volume_server_pb.VolumeServer/SetState" - VolumeServer_VolumeCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeCopy" - VolumeServer_ReadVolumeFileStatus_FullMethodName = "/volume_server_pb.VolumeServer/ReadVolumeFileStatus" - VolumeServer_CopyFile_FullMethodName = "/volume_server_pb.VolumeServer/CopyFile" - VolumeServer_ReceiveFile_FullMethodName = "/volume_server_pb.VolumeServer/ReceiveFile" - VolumeServer_ReadNeedleBlob_FullMethodName = "/volume_server_pb.VolumeServer/ReadNeedleBlob" - VolumeServer_ReadNeedleMeta_FullMethodName = "/volume_server_pb.VolumeServer/ReadNeedleMeta" - VolumeServer_WriteNeedleBlob_FullMethodName = "/volume_server_pb.VolumeServer/WriteNeedleBlob" - VolumeServer_ReadAllNeedles_FullMethodName = "/volume_server_pb.VolumeServer/ReadAllNeedles" - VolumeServer_VolumeTailSender_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTailSender" - VolumeServer_VolumeTailReceiver_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTailReceiver" - VolumeServer_VolumeEcShardsGenerate_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate" - VolumeServer_VolumeEcShardsRebuild_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild" - VolumeServer_VolumeEcShardsCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsCopy" - VolumeServer_VolumeEcShardsDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsDelete" - VolumeServer_VolumeEcShardsMount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsMount" - VolumeServer_VolumeEcShardsUnmount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount" - VolumeServer_VolumeEcShardRead_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardRead" - VolumeServer_VolumeEcBlobDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcBlobDelete" - VolumeServer_VolumeEcShardsToVolume_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume" - VolumeServer_VolumeEcShardsInfo_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsInfo" - VolumeServer_VolumeTierMoveDatToRemote_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTierMoveDatToRemote" - VolumeServer_VolumeTierMoveDatFromRemote_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTierMoveDatFromRemote" - VolumeServer_VolumeServerStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerStatus" - VolumeServer_VolumeServerLeave_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerLeave" - VolumeServer_FetchAndWriteNeedle_FullMethodName = "/volume_server_pb.VolumeServer/FetchAndWriteNeedle" - VolumeServer_ScrubVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubVolume" - VolumeServer_ScrubEcVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubEcVolume" - VolumeServer_Query_FullMethodName = "/volume_server_pb.VolumeServer/Query" - VolumeServer_VolumeNeedleStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeNeedleStatus" - VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping" + VolumeServer_BatchDelete_FullMethodName = "/volume_server_pb.VolumeServer/BatchDelete" + VolumeServer_VacuumVolumeCheck_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCheck" + VolumeServer_VacuumVolumeCompact_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCompact" + VolumeServer_VacuumVolumeCommit_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCommit" + VolumeServer_VacuumVolumeCleanup_FullMethodName = "/volume_server_pb.VolumeServer/VacuumVolumeCleanup" + VolumeServer_DeleteCollection_FullMethodName = "/volume_server_pb.VolumeServer/DeleteCollection" + VolumeServer_AllocateVolume_FullMethodName = "/volume_server_pb.VolumeServer/AllocateVolume" + VolumeServer_VolumeSyncStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeSyncStatus" + VolumeServer_VolumeIncrementalCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeIncrementalCopy" + VolumeServer_VolumeMount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMount" + VolumeServer_VolumeUnmount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeUnmount" + VolumeServer_VolumeDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeDelete" + VolumeServer_VolumeMarkReadonly_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMarkReadonly" + VolumeServer_VolumeMarkWritable_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMarkWritable" + VolumeServer_VolumeConfigure_FullMethodName = "/volume_server_pb.VolumeServer/VolumeConfigure" + VolumeServer_VolumeStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeStatus" + VolumeServer_GetState_FullMethodName = "/volume_server_pb.VolumeServer/GetState" + VolumeServer_SetState_FullMethodName = "/volume_server_pb.VolumeServer/SetState" + VolumeServer_VolumeCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeCopy" + VolumeServer_ReadVolumeFileStatus_FullMethodName = "/volume_server_pb.VolumeServer/ReadVolumeFileStatus" + VolumeServer_CopyFile_FullMethodName = "/volume_server_pb.VolumeServer/CopyFile" + VolumeServer_ReceiveFile_FullMethodName = "/volume_server_pb.VolumeServer/ReceiveFile" + VolumeServer_ReadNeedleBlob_FullMethodName = "/volume_server_pb.VolumeServer/ReadNeedleBlob" + VolumeServer_ReadNeedleMeta_FullMethodName = "/volume_server_pb.VolumeServer/ReadNeedleMeta" + VolumeServer_WriteNeedleBlob_FullMethodName = "/volume_server_pb.VolumeServer/WriteNeedleBlob" + VolumeServer_ReadAllNeedles_FullMethodName = "/volume_server_pb.VolumeServer/ReadAllNeedles" + VolumeServer_VolumeTailSender_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTailSender" + VolumeServer_VolumeTailReceiver_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTailReceiver" + VolumeServer_VolumeEcShardsGenerate_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsGenerate" + VolumeServer_VolumeEcShardsRebuild_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsRebuild" + VolumeServer_VolumeEcShardsCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsCopy" + VolumeServer_VolumeEcShardsDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsDelete" + VolumeServer_VolumeEcShardsMount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsMount" + VolumeServer_VolumeEcShardsUnmount_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsUnmount" + VolumeServer_VolumeEcShardRead_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardRead" + VolumeServer_VolumeEcBlobDelete_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcBlobDelete" + VolumeServer_VolumeEcShardsToVolume_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsToVolume" + VolumeServer_VolumeEcShardsInfo_FullMethodName = "/volume_server_pb.VolumeServer/VolumeEcShardsInfo" + VolumeServer_VolumeTierMoveDatToRemote_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTierMoveDatToRemote" + VolumeServer_VolumeTierMoveDatFromRemote_FullMethodName = "/volume_server_pb.VolumeServer/VolumeTierMoveDatFromRemote" + VolumeServer_VolumeServerStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerStatus" + VolumeServer_VolumeServerLeave_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerLeave" + VolumeServer_FetchAndWriteNeedle_FullMethodName = "/volume_server_pb.VolumeServer/FetchAndWriteNeedle" + VolumeServer_ScrubVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubVolume" + VolumeServer_ScrubEcVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubEcVolume" + VolumeServer_Query_FullMethodName = "/volume_server_pb.VolumeServer/Query" + VolumeServer_VolumeNeedleStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeNeedleStatus" + VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping" + VolumeServer_AllocateBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/AllocateBlockVolume" + VolumeServer_VolumeServerDeleteBlockVolume_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerDeleteBlockVolume" ) // VolumeServerClient is the client API for VolumeServer service. @@ -128,6 +130,9 @@ type VolumeServerClient interface { Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueriedStripe], error) VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) + // block volumes + AllocateBlockVolume(ctx context.Context, in *AllocateBlockVolumeRequest, opts ...grpc.CallOption) (*AllocateBlockVolumeResponse, error) + VolumeServerDeleteBlockVolume(ctx context.Context, in *VolumeServerDeleteBlockVolumeRequest, opts ...grpc.CallOption) (*VolumeServerDeleteBlockVolumeResponse, error) } type volumeServerClient struct { @@ -711,6 +716,26 @@ func (c *volumeServerClient) Ping(ctx context.Context, in *PingRequest, opts ... return out, nil } +func (c *volumeServerClient) AllocateBlockVolume(ctx context.Context, in *AllocateBlockVolumeRequest, opts ...grpc.CallOption) (*AllocateBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AllocateBlockVolumeResponse) + err := c.cc.Invoke(ctx, VolumeServer_AllocateBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *volumeServerClient) VolumeServerDeleteBlockVolume(ctx context.Context, in *VolumeServerDeleteBlockVolumeRequest, opts ...grpc.CallOption) (*VolumeServerDeleteBlockVolumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(VolumeServerDeleteBlockVolumeResponse) + err := c.cc.Invoke(ctx, VolumeServer_VolumeServerDeleteBlockVolume_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // VolumeServerServer is the server API for VolumeServer service. // All implementations must embed UnimplementedVolumeServerServer // for forward compatibility. @@ -770,6 +795,9 @@ type VolumeServerServer interface { Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) Ping(context.Context, *PingRequest) (*PingResponse, error) + // block volumes + AllocateBlockVolume(context.Context, *AllocateBlockVolumeRequest) (*AllocateBlockVolumeResponse, error) + VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error) mustEmbedUnimplementedVolumeServerServer() } @@ -781,148 +809,154 @@ type VolumeServerServer interface { type UnimplementedVolumeServerServer struct{} func (UnimplementedVolumeServerServer) BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method BatchDelete not implemented") + return nil, status.Error(codes.Unimplemented, "method BatchDelete not implemented") } func (UnimplementedVolumeServerServer) VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCheck not implemented") + return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCheck not implemented") } func (UnimplementedVolumeServerServer) VacuumVolumeCompact(*VacuumVolumeCompactRequest, grpc.ServerStreamingServer[VacuumVolumeCompactResponse]) error { - return status.Errorf(codes.Unimplemented, "method VacuumVolumeCompact not implemented") + return status.Error(codes.Unimplemented, "method VacuumVolumeCompact not implemented") } func (UnimplementedVolumeServerServer) VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCommit not implemented") + return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCommit not implemented") } func (UnimplementedVolumeServerServer) VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCleanup not implemented") + return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCleanup not implemented") } func (UnimplementedVolumeServerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") + return nil, status.Error(codes.Unimplemented, "method DeleteCollection not implemented") } func (UnimplementedVolumeServerServer) AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AllocateVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method AllocateVolume not implemented") } func (UnimplementedVolumeServerServer) VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeSyncStatus not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeSyncStatus not implemented") } func (UnimplementedVolumeServerServer) VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, grpc.ServerStreamingServer[VolumeIncrementalCopyResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeIncrementalCopy not implemented") + return status.Error(codes.Unimplemented, "method VolumeIncrementalCopy not implemented") } func (UnimplementedVolumeServerServer) VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMount not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeMount not implemented") } func (UnimplementedVolumeServerServer) VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeUnmount not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeUnmount not implemented") } func (UnimplementedVolumeServerServer) VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeDelete not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeDelete not implemented") } func (UnimplementedVolumeServerServer) VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkReadonly not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeMarkReadonly not implemented") } func (UnimplementedVolumeServerServer) VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkWritable not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeMarkWritable not implemented") } func (UnimplementedVolumeServerServer) VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeConfigure not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeConfigure not implemented") } func (UnimplementedVolumeServerServer) VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeStatus not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeStatus not implemented") } func (UnimplementedVolumeServerServer) GetState(context.Context, *GetStateRequest) (*GetStateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetState not implemented") + return nil, status.Error(codes.Unimplemented, "method GetState not implemented") } func (UnimplementedVolumeServerServer) SetState(context.Context, *SetStateRequest) (*SetStateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method SetState not implemented") + return nil, status.Error(codes.Unimplemented, "method SetState not implemented") } func (UnimplementedVolumeServerServer) VolumeCopy(*VolumeCopyRequest, grpc.ServerStreamingServer[VolumeCopyResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeCopy not implemented") + return status.Error(codes.Unimplemented, "method VolumeCopy not implemented") } func (UnimplementedVolumeServerServer) ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReadVolumeFileStatus not implemented") + return nil, status.Error(codes.Unimplemented, "method ReadVolumeFileStatus not implemented") } func (UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, grpc.ServerStreamingServer[CopyFileResponse]) error { - return status.Errorf(codes.Unimplemented, "method CopyFile not implemented") + return status.Error(codes.Unimplemented, "method CopyFile not implemented") } func (UnimplementedVolumeServerServer) ReceiveFile(grpc.ClientStreamingServer[ReceiveFileRequest, ReceiveFileResponse]) error { - return status.Errorf(codes.Unimplemented, "method ReceiveFile not implemented") + return status.Error(codes.Unimplemented, "method ReceiveFile not implemented") } func (UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented") + return nil, status.Error(codes.Unimplemented, "method ReadNeedleBlob not implemented") } func (UnimplementedVolumeServerServer) ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleMeta not implemented") + return nil, status.Error(codes.Unimplemented, "method ReadNeedleMeta not implemented") } func (UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented") + return nil, status.Error(codes.Unimplemented, "method WriteNeedleBlob not implemented") } func (UnimplementedVolumeServerServer) ReadAllNeedles(*ReadAllNeedlesRequest, grpc.ServerStreamingServer[ReadAllNeedlesResponse]) error { - return status.Errorf(codes.Unimplemented, "method ReadAllNeedles not implemented") + return status.Error(codes.Unimplemented, "method ReadAllNeedles not implemented") } func (UnimplementedVolumeServerServer) VolumeTailSender(*VolumeTailSenderRequest, grpc.ServerStreamingServer[VolumeTailSenderResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeTailSender not implemented") + return status.Error(codes.Unimplemented, "method VolumeTailSender not implemented") } func (UnimplementedVolumeServerServer) VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeTailReceiver not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeTailReceiver not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsCopy not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsCopy not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsDelete not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsDelete not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsMount not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsMount not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardRead(*VolumeEcShardReadRequest, grpc.ServerStreamingServer[VolumeEcShardReadResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeEcShardRead not implemented") + return status.Error(codes.Unimplemented, "method VolumeEcShardRead not implemented") } func (UnimplementedVolumeServerServer) VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcBlobDelete not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcBlobDelete not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented") } func (UnimplementedVolumeServerServer) VolumeEcShardsInfo(context.Context, *VolumeEcShardsInfoRequest) (*VolumeEcShardsInfoResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsInfo not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsInfo not implemented") } func (UnimplementedVolumeServerServer) VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, grpc.ServerStreamingServer[VolumeTierMoveDatToRemoteResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented") + return status.Error(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented") } func (UnimplementedVolumeServerServer) VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, grpc.ServerStreamingServer[VolumeTierMoveDatFromRemoteResponse]) error { - return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented") + return status.Error(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented") } func (UnimplementedVolumeServerServer) VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeServerStatus not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeServerStatus not implemented") } func (UnimplementedVolumeServerServer) VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeServerLeave not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeServerLeave not implemented") } func (UnimplementedVolumeServerServer) FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method FetchAndWriteNeedle not implemented") + return nil, status.Error(codes.Unimplemented, "method FetchAndWriteNeedle not implemented") } func (UnimplementedVolumeServerServer) ScrubVolume(context.Context, *ScrubVolumeRequest) (*ScrubVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ScrubVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method ScrubVolume not implemented") } func (UnimplementedVolumeServerServer) ScrubEcVolume(context.Context, *ScrubEcVolumeRequest) (*ScrubEcVolumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ScrubEcVolume not implemented") + return nil, status.Error(codes.Unimplemented, "method ScrubEcVolume not implemented") } func (UnimplementedVolumeServerServer) Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error { - return status.Errorf(codes.Unimplemented, "method Query not implemented") + return status.Error(codes.Unimplemented, "method Query not implemented") } func (UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented") + return nil, status.Error(codes.Unimplemented, "method VolumeNeedleStatus not implemented") } func (UnimplementedVolumeServerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") + return nil, status.Error(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedVolumeServerServer) AllocateBlockVolume(context.Context, *AllocateBlockVolumeRequest) (*AllocateBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AllocateBlockVolume not implemented") +} +func (UnimplementedVolumeServerServer) VolumeServerDeleteBlockVolume(context.Context, *VolumeServerDeleteBlockVolumeRequest) (*VolumeServerDeleteBlockVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "method VolumeServerDeleteBlockVolume not implemented") } func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {} func (UnimplementedVolumeServerServer) testEmbeddedByValue() {} @@ -935,7 +969,7 @@ type UnsafeVolumeServerServer interface { } func RegisterVolumeServerServer(s grpc.ServiceRegistrar, srv VolumeServerServer) { - // If the following call pancis, it indicates UnimplementedVolumeServerServer was + // If the following call panics, it indicates UnimplementedVolumeServerServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. @@ -1728,6 +1762,42 @@ func _VolumeServer_Ping_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _VolumeServer_AllocateBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AllocateBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).AllocateBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_AllocateBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).AllocateBlockVolume(ctx, req.(*AllocateBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _VolumeServer_VolumeServerDeleteBlockVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeServerDeleteBlockVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeServerDeleteBlockVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: VolumeServer_VolumeServerDeleteBlockVolume_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeServerDeleteBlockVolume(ctx, req.(*VolumeServerDeleteBlockVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + // VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1883,6 +1953,14 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{ MethodName: "Ping", Handler: _VolumeServer_Ping_Handler, }, + { + MethodName: "AllocateBlockVolume", + Handler: _VolumeServer_AllocateBlockVolume_Handler, + }, + { + MethodName: "VolumeServerDeleteBlockVolume", + Handler: _VolumeServer_VolumeServerDeleteBlockVolume_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go new file mode 100644 index 000000000..d0c6abb36 --- /dev/null +++ b/weed/server/master_block_registry.go @@ -0,0 +1,276 @@ +package weed_server + +import ( + "fmt" + "sync" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// VolumeStatus tracks the lifecycle of a block volume entry. +type VolumeStatus int + +const ( + StatusPending VolumeStatus = iota // Created via RPC, not yet confirmed by heartbeat + StatusActive // Confirmed by heartbeat from volume server +) + +// BlockVolumeEntry tracks one block volume across the cluster. +type BlockVolumeEntry struct { + Name string + VolumeServer string // volume server address (ip:port or grpc addr) + Path string // file path on volume server + IQN string + ISCSIAddr string + SizeBytes uint64 + Epoch uint64 + Role uint32 + Status VolumeStatus +} + +// BlockVolumeRegistry is the in-memory registry of block volumes. +// Rebuilt from heartbeats on master restart (no persistence). +type BlockVolumeRegistry struct { + mu sync.RWMutex + volumes map[string]*BlockVolumeEntry // keyed by name + byServer map[string]map[string]bool // server -> set of volume names + blockServers map[string]bool // servers known to support block volumes + + // inflight guards concurrent CreateBlockVolume for the same name. + inflight sync.Map // name -> *inflightEntry +} + +type inflightEntry struct{} + +// NewBlockVolumeRegistry creates an empty registry. +func NewBlockVolumeRegistry() *BlockVolumeRegistry { + return &BlockVolumeRegistry{ + volumes: make(map[string]*BlockVolumeEntry), + byServer: make(map[string]map[string]bool), + blockServers: make(map[string]bool), + } +} + +// Register adds an entry to the registry. +// Returns error if a volume with the same name already exists. +func (r *BlockVolumeRegistry) Register(entry *BlockVolumeEntry) error { + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.volumes[entry.Name]; ok { + return fmt.Errorf("block volume %q already registered", entry.Name) + } + r.volumes[entry.Name] = entry + r.addToServer(entry.VolumeServer, entry.Name) + return nil +} + +// Unregister removes and returns the entry. Returns nil if not found. +func (r *BlockVolumeRegistry) Unregister(name string) *BlockVolumeEntry { + r.mu.Lock() + defer r.mu.Unlock() + entry, ok := r.volumes[name] + if !ok { + return nil + } + delete(r.volumes, name) + r.removeFromServer(entry.VolumeServer, name) + return entry +} + +// Lookup returns the entry for the given name. +func (r *BlockVolumeRegistry) Lookup(name string) (*BlockVolumeEntry, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + e, ok := r.volumes[name] + return e, ok +} + +// ListByServer returns all entries hosted on the given server. +func (r *BlockVolumeRegistry) ListByServer(server string) []*BlockVolumeEntry { + r.mu.RLock() + defer r.mu.RUnlock() + names, ok := r.byServer[server] + if !ok { + return nil + } + entries := make([]*BlockVolumeEntry, 0, len(names)) + for name := range names { + if e, ok := r.volumes[name]; ok { + entries = append(entries, e) + } + } + return entries +} + +// UpdateFullHeartbeat reconciles the registry from a full heartbeat. +// Called on the first heartbeat from a volume server. +// Marks reported volumes as Active, removes entries for this server +// that are not reported (stale). +func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master_pb.BlockVolumeInfoMessage) { + r.mu.Lock() + defer r.mu.Unlock() + + // Mark server as block-capable since it sent block volume info. + r.blockServers[server] = true + + // Build set of reported paths. + reported := make(map[string]*master_pb.BlockVolumeInfoMessage, len(infos)) + for _, info := range infos { + reported[info.Path] = info + } + + // Find entries for this server that are NOT reported -> remove them. + if names, ok := r.byServer[server]; ok { + for name := range names { + entry := r.volumes[name] + if entry == nil { + continue + } + if _, found := reported[entry.Path]; !found { + delete(r.volumes, name) + delete(names, name) + } + } + } + + // Update or add entries for reported volumes. + for _, info := range infos { + // Find existing entry by path on this server. + var existing *BlockVolumeEntry + if names, ok := r.byServer[server]; ok { + for name := range names { + if e := r.volumes[name]; e != nil && e.Path == info.Path { + existing = e + break + } + } + } + if existing != nil { + // Update fields from heartbeat. + existing.SizeBytes = info.VolumeSize + existing.Epoch = info.Epoch + existing.Role = info.Role + existing.Status = StatusActive + } + // If no existing entry found by path, it was created outside master + // (e.g., manually). We don't auto-register unknown volumes — they + // must be created via CreateBlockVolume RPC. + } +} + +// UpdateDeltaHeartbeat processes incremental new/deleted block volumes. +// Called on subsequent heartbeats (not the first). +func (r *BlockVolumeRegistry) UpdateDeltaHeartbeat(server string, added []*master_pb.BlockVolumeShortInfoMessage, removed []*master_pb.BlockVolumeShortInfoMessage) { + r.mu.Lock() + defer r.mu.Unlock() + + // Remove deleted volumes. + for _, rm := range removed { + if names, ok := r.byServer[server]; ok { + for name := range names { + if e := r.volumes[name]; e != nil && e.Path == rm.Path { + delete(r.volumes, name) + delete(names, name) + break + } + } + } + } + + // Mark newly appeared volumes as active (if they exist in registry). + for _, add := range added { + if names, ok := r.byServer[server]; ok { + for name := range names { + if e := r.volumes[name]; e != nil && e.Path == add.Path { + e.Status = StatusActive + break + } + } + } + } +} + +// PickServer returns the server address with the fewest block volumes. +// servers is the list of online volume server addresses. +// Returns error if no servers available. +func (r *BlockVolumeRegistry) PickServer(servers []string) (string, error) { + if len(servers) == 0 { + return "", fmt.Errorf("no block volume servers available") + } + r.mu.RLock() + defer r.mu.RUnlock() + + best := servers[0] + bestCount := r.countForServer(best) + for _, s := range servers[1:] { + c := r.countForServer(s) + if c < bestCount { + best = s + bestCount = c + } + } + return best, nil +} + +// AcquireInflight tries to acquire a per-name create lock. +// Returns true if acquired (caller must call ReleaseInflight when done). +// Returns false if another create is already in progress for this name. +func (r *BlockVolumeRegistry) AcquireInflight(name string) bool { + _, loaded := r.inflight.LoadOrStore(name, &inflightEntry{}) + return !loaded // true = we stored it (acquired), false = already existed +} + +// ReleaseInflight releases the per-name create lock. +func (r *BlockVolumeRegistry) ReleaseInflight(name string) { + r.inflight.Delete(name) +} + +// countForServer returns the number of volumes on the given server. +// Caller must hold at least RLock. +func (r *BlockVolumeRegistry) countForServer(server string) int { + if names, ok := r.byServer[server]; ok { + return len(names) + } + return 0 +} + +func (r *BlockVolumeRegistry) addToServer(server, name string) { + if r.byServer[server] == nil { + r.byServer[server] = make(map[string]bool) + } + r.byServer[server][name] = true +} + +func (r *BlockVolumeRegistry) removeFromServer(server, name string) { + if names, ok := r.byServer[server]; ok { + delete(names, name) + if len(names) == 0 { + delete(r.byServer, server) + } + } +} + +// MarkBlockCapable records that the given server supports block volumes. +func (r *BlockVolumeRegistry) MarkBlockCapable(server string) { + r.mu.Lock() + r.blockServers[server] = true + r.mu.Unlock() +} + +// UnmarkBlockCapable removes a server from the block-capable set. +func (r *BlockVolumeRegistry) UnmarkBlockCapable(server string) { + r.mu.Lock() + delete(r.blockServers, server) + r.mu.Unlock() +} + +// BlockCapableServers returns the list of servers known to support block volumes. +func (r *BlockVolumeRegistry) BlockCapableServers() []string { + r.mu.RLock() + defer r.mu.RUnlock() + servers := make([]string, 0, len(r.blockServers)) + for s := range r.blockServers { + servers = append(servers, s) + } + return servers +} diff --git a/weed/server/master_block_registry_test.go b/weed/server/master_block_registry_test.go new file mode 100644 index 000000000..9557039c6 --- /dev/null +++ b/weed/server/master_block_registry_test.go @@ -0,0 +1,292 @@ +package weed_server + +import ( + "fmt" + "sync" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestRegistry_RegisterLookup(t *testing.T) { + r := NewBlockVolumeRegistry() + entry := &BlockVolumeEntry{ + Name: "vol1", + VolumeServer: "server1:9333", + Path: "/data/vol1.blk", + IQN: "iqn.2024.com.seaweedfs:vol1", + ISCSIAddr: "10.0.0.1:3260", + SizeBytes: 1 << 30, + Epoch: 1, + Role: 1, + Status: StatusPending, + } + if err := r.Register(entry); err != nil { + t.Fatalf("Register: %v", err) + } + got, ok := r.Lookup("vol1") + if !ok { + t.Fatal("Lookup: not found") + } + if got.Name != "vol1" || got.VolumeServer != "server1:9333" || got.Path != "/data/vol1.blk" { + t.Fatalf("Lookup: unexpected entry: %+v", got) + } + if got.Status != StatusPending { + t.Fatalf("Status: got %d, want %d", got.Status, StatusPending) + } +} + +func TestRegistry_Unregister(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/vol1.blk"}) + removed := r.Unregister("vol1") + if removed == nil { + t.Fatal("Unregister returned nil") + } + if _, ok := r.Lookup("vol1"); ok { + t.Fatal("vol1 should not be found after Unregister") + } + // Double unregister returns nil. + if r.Unregister("vol1") != nil { + t.Fatal("double Unregister should return nil") + } +} + +func TestRegistry_DuplicateRegister(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/vol1.blk"}) + err := r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s2", Path: "/vol1.blk"}) + if err == nil { + t.Fatal("duplicate Register should return error") + } +} + +func TestRegistry_ListByServer(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk"}) + r.Register(&BlockVolumeEntry{Name: "vol3", VolumeServer: "s2", Path: "/v3.blk"}) + + s1Vols := r.ListByServer("s1") + if len(s1Vols) != 2 { + t.Fatalf("ListByServer(s1): got %d, want 2", len(s1Vols)) + } + s2Vols := r.ListByServer("s2") + if len(s2Vols) != 1 { + t.Fatalf("ListByServer(s2): got %d, want 1", len(s2Vols)) + } + s3Vols := r.ListByServer("s3") + if len(s3Vols) != 0 { + t.Fatalf("ListByServer(s3): got %d, want 0", len(s3Vols)) + } +} + +func TestRegistry_UpdateFullHeartbeat(t *testing.T) { + r := NewBlockVolumeRegistry() + // Register two volumes on server s1. + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending}) + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", Status: StatusPending}) + + // Full heartbeat reports only vol1 (vol2 is stale). + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/v1.blk", Epoch: 5, Role: 1}, + }) + + // vol1 should be Active. + e1, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should exist after full heartbeat") + } + if e1.Status != StatusActive { + t.Fatalf("vol1 status: got %d, want %d", e1.Status, StatusActive) + } + if e1.Epoch != 5 { + t.Fatalf("vol1 epoch: got %d, want 5", e1.Epoch) + } + + // vol2 should be removed (stale). + if _, ok := r.Lookup("vol2"); ok { + t.Fatal("vol2 should have been removed as stale") + } +} + +func TestRegistry_UpdateDeltaHeartbeat(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending}) + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", Status: StatusActive}) + + // Delta: vol1 newly appeared, vol2 deleted. + r.UpdateDeltaHeartbeat("s1", + []*master_pb.BlockVolumeShortInfoMessage{{Path: "/v1.blk"}}, + []*master_pb.BlockVolumeShortInfoMessage{{Path: "/v2.blk"}}, + ) + + // vol1 should be Active. + e1, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should exist") + } + if e1.Status != StatusActive { + t.Fatalf("vol1 status: got %d, want Active", e1.Status) + } + + // vol2 should be removed. + if _, ok := r.Lookup("vol2"); ok { + t.Fatal("vol2 should have been removed by delta") + } +} + +func TestRegistry_PendingToActive(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", + Status: StatusPending, Epoch: 1, + }) + + // Full heartbeat confirms the volume. + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/v1.blk", Epoch: 1, Role: 1}, + }) + + e, _ := r.Lookup("vol1") + if e.Status != StatusActive { + t.Fatalf("expected Active after heartbeat, got %d", e.Status) + } +} + +func TestRegistry_PickServer(t *testing.T) { + r := NewBlockVolumeRegistry() + // s1 has 2 volumes, s2 has 1, s3 has 0. + r.Register(&BlockVolumeEntry{Name: "v1", VolumeServer: "s1", Path: "/v1.blk"}) + r.Register(&BlockVolumeEntry{Name: "v2", VolumeServer: "s1", Path: "/v2.blk"}) + r.Register(&BlockVolumeEntry{Name: "v3", VolumeServer: "s2", Path: "/v3.blk"}) + + got, err := r.PickServer([]string{"s1", "s2", "s3"}) + if err != nil { + t.Fatalf("PickServer: %v", err) + } + if got != "s3" { + t.Fatalf("PickServer: got %q, want s3 (fewest volumes)", got) + } +} + +func TestRegistry_PickServerEmpty(t *testing.T) { + r := NewBlockVolumeRegistry() + _, err := r.PickServer(nil) + if err == nil { + t.Fatal("PickServer with no servers should return error") + } +} + +func TestRegistry_InflightLock(t *testing.T) { + r := NewBlockVolumeRegistry() + + // First acquire succeeds. + if !r.AcquireInflight("vol1") { + t.Fatal("first AcquireInflight should succeed") + } + + // Second acquire for same name fails. + if r.AcquireInflight("vol1") { + t.Fatal("second AcquireInflight for same name should fail") + } + + // Different name succeeds. + if !r.AcquireInflight("vol2") { + t.Fatal("AcquireInflight for different name should succeed") + } + + // Release and re-acquire. + r.ReleaseInflight("vol1") + if !r.AcquireInflight("vol1") { + t.Fatal("AcquireInflight after release should succeed") + } + + r.ReleaseInflight("vol1") + r.ReleaseInflight("vol2") +} + +func TestRegistry_UnmarkDeadServer(t *testing.T) { + r := NewBlockVolumeRegistry() + r.MarkBlockCapable("s1") + r.MarkBlockCapable("s2") + + servers := r.BlockCapableServers() + if len(servers) != 2 { + t.Fatalf("expected 2 servers, got %d", len(servers)) + } + + // Simulate s1 disconnect. + r.UnmarkBlockCapable("s1") + + servers = r.BlockCapableServers() + if len(servers) != 1 { + t.Fatalf("expected 1 server after unmark, got %d", len(servers)) + } + if servers[0] != "s2" { + t.Fatalf("expected s2, got %s", servers[0]) + } +} + +func TestRegistry_FullHeartbeatUpdatesSizeBytes(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{ + Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", + SizeBytes: 1 << 30, Status: StatusPending, + }) + + // Heartbeat with updated size (online resize). + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/v1.blk", VolumeSize: 2 << 30, Epoch: 1, Role: 1}, + }) + + e, _ := r.Lookup("vol1") + if e.SizeBytes != 2<<30 { + t.Fatalf("SizeBytes: got %d, want %d", e.SizeBytes, 2<<30) + } +} + +func TestRegistry_ConcurrentAccess(t *testing.T) { + r := NewBlockVolumeRegistry() + var wg sync.WaitGroup + n := 50 + + // Concurrent register. + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + name := fmt.Sprintf("vol%d", i) + r.Register(&BlockVolumeEntry{ + Name: name, VolumeServer: "s1", + Path: fmt.Sprintf("/v%d.blk", i), + }) + }(i) + } + wg.Wait() + + // All should be findable. + for i := 0; i < n; i++ { + name := fmt.Sprintf("vol%d", i) + if _, ok := r.Lookup(name); !ok { + t.Fatalf("vol%d not found after concurrent register", i) + } + } + + // Concurrent unregister. + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r.Unregister(fmt.Sprintf("vol%d", i)) + }(i) + } + wg.Wait() + + // All should be gone. + for i := 0; i < n; i++ { + if _, ok := r.Lookup(fmt.Sprintf("vol%d", i)); ok { + t.Fatalf("vol%d found after concurrent unregister", i) + } + } +} diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 395eb4ebd..b8534553b 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -89,6 +89,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ ms.Topo.UnRegisterDataNode(dn) glog.V(0).Infof("unregister disconnected volume server %s:%d", dn.Ip, dn.Port) ms.UnRegisterUuids(dn.Ip, dn.Port) + if ms.blockRegistry != nil { + ms.blockRegistry.UnmarkBlockCapable(dn.Url()) + } if ms.Topo.IsLeader() && (len(message.DeletedVids) > 0 || len(message.DeletedEcVids) > 0) { ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message}) @@ -263,6 +266,16 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 || len(message.NewEcVids) > 0 || len(message.DeletedEcVids) > 0 { ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message}) } + + // Process block volume heartbeat. + // These are mutually exclusive: the volume server sends either a full list + // (BlockVolumeInfos on first heartbeat) or deltas (NewBlockVolumes/DeletedBlockVolumes + // on subsequent heartbeats), never both in the same message. + if len(heartbeat.BlockVolumeInfos) > 0 || heartbeat.HasNoBlockVolumes { + ms.blockRegistry.UpdateFullHeartbeat(dn.Url(), heartbeat.BlockVolumeInfos) + } else if len(heartbeat.NewBlockVolumes) > 0 || len(heartbeat.DeletedBlockVolumes) > 0 { + ms.blockRegistry.UpdateDeltaHeartbeat(dn.Url(), heartbeat.NewBlockVolumes, heartbeat.DeletedBlockVolumes) + } } } diff --git a/weed/server/master_grpc_server_block.go b/weed/server/master_grpc_server_block.go new file mode 100644 index 000000000..232912444 --- /dev/null +++ b/weed/server/master_grpc_server_block.go @@ -0,0 +1,167 @@ +package weed_server + +import ( + "context" + "fmt" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// CreateBlockVolume picks a volume server, delegates creation, and records +// the mapping in the block volume registry. +func (ms *MasterServer) CreateBlockVolume(ctx context.Context, req *master_pb.CreateBlockVolumeRequest) (*master_pb.CreateBlockVolumeResponse, error) { + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + if req.SizeBytes == 0 { + return nil, fmt.Errorf("size_bytes must be > 0") + } + + // Idempotent: if already registered, return existing entry (validate size). + if entry, ok := ms.blockRegistry.Lookup(req.Name); ok { + if entry.SizeBytes < req.SizeBytes { + return nil, fmt.Errorf("block volume %q exists with size %d (requested %d)", req.Name, entry.SizeBytes, req.SizeBytes) + } + return &master_pb.CreateBlockVolumeResponse{ + VolumeId: entry.Name, + VolumeServer: entry.VolumeServer, + IscsiAddr: entry.ISCSIAddr, + Iqn: entry.IQN, + CapacityBytes: entry.SizeBytes, + }, nil + } + + // Per-name inflight lock prevents concurrent creates for the same name. + if !ms.blockRegistry.AcquireInflight(req.Name) { + return nil, fmt.Errorf("block volume %q creation already in progress", req.Name) + } + defer ms.blockRegistry.ReleaseInflight(req.Name) + + // Double-check after acquiring lock (another goroutine may have finished). + if entry, ok := ms.blockRegistry.Lookup(req.Name); ok { + return &master_pb.CreateBlockVolumeResponse{ + VolumeId: entry.Name, + VolumeServer: entry.VolumeServer, + IscsiAddr: entry.ISCSIAddr, + Iqn: entry.IQN, + CapacityBytes: entry.SizeBytes, + }, nil + } + + // Get candidate servers. + servers := ms.blockRegistry.BlockCapableServers() + if len(servers) == 0 { + return nil, fmt.Errorf("no block volume servers available") + } + + // Try up to 3 servers (or all available, whichever is smaller). + maxRetries := 3 + if len(servers) < maxRetries { + maxRetries = len(servers) + } + + var lastErr error + for attempt := 0; attempt < maxRetries; attempt++ { + server, err := ms.blockRegistry.PickServer(servers) + if err != nil { + return nil, err + } + + path, iqn, iscsiAddr, err := ms.blockVSAllocate(ctx, pb.ServerAddress(server), req.Name, req.SizeBytes, req.DiskType) + if err != nil { + lastErr = fmt.Errorf("server %s: %w", server, err) + glog.V(0).Infof("CreateBlockVolume %q: attempt %d on %s failed: %v", req.Name, attempt+1, server, err) + servers = removeServer(servers, server) + continue + } + + // Register in registry as Active (VS confirmed creation). + // Heartbeat will update epoch/role fields later. + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: req.Name, + VolumeServer: server, + Path: path, + IQN: iqn, + ISCSIAddr: iscsiAddr, + SizeBytes: req.SizeBytes, + Status: StatusActive, + }); err != nil { + // Already registered (race condition) — return the existing entry. + if existing, ok := ms.blockRegistry.Lookup(req.Name); ok { + return &master_pb.CreateBlockVolumeResponse{ + VolumeId: existing.Name, + VolumeServer: existing.VolumeServer, + IscsiAddr: existing.ISCSIAddr, + Iqn: existing.IQN, + CapacityBytes: existing.SizeBytes, + }, nil + } + return nil, fmt.Errorf("register block volume: %w", err) + } + + glog.V(0).Infof("CreateBlockVolume %q: created on %s (path=%s, iqn=%s)", req.Name, server, path, iqn) + return &master_pb.CreateBlockVolumeResponse{ + VolumeId: req.Name, + VolumeServer: server, + IscsiAddr: iscsiAddr, + Iqn: iqn, + CapacityBytes: req.SizeBytes, + }, nil + } + + return nil, fmt.Errorf("all volume servers failed for %q: %v", req.Name, lastErr) +} + +// DeleteBlockVolume removes a block volume from the registry and volume server. +func (ms *MasterServer) DeleteBlockVolume(ctx context.Context, req *master_pb.DeleteBlockVolumeRequest) (*master_pb.DeleteBlockVolumeResponse, error) { + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + entry, ok := ms.blockRegistry.Lookup(req.Name) + if !ok { + // Idempotent: not found is success. + return &master_pb.DeleteBlockVolumeResponse{}, nil + } + + // Call volume server to delete. + if err := ms.blockVSDelete(ctx, pb.ServerAddress(entry.VolumeServer), req.Name); err != nil { + return nil, fmt.Errorf("delete block volume %q on %s: %w", req.Name, entry.VolumeServer, err) + } + + ms.blockRegistry.Unregister(req.Name) + glog.V(0).Infof("DeleteBlockVolume %q: removed from %s", req.Name, entry.VolumeServer) + return &master_pb.DeleteBlockVolumeResponse{}, nil +} + +// LookupBlockVolume looks up a block volume in the registry. +func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.LookupBlockVolumeRequest) (*master_pb.LookupBlockVolumeResponse, error) { + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + entry, ok := ms.blockRegistry.Lookup(req.Name) + if !ok { + return nil, fmt.Errorf("block volume %q not found", req.Name) + } + + return &master_pb.LookupBlockVolumeResponse{ + VolumeServer: entry.VolumeServer, + IscsiAddr: entry.ISCSIAddr, + Iqn: entry.IQN, + CapacityBytes: entry.SizeBytes, + }, nil +} + +// removeServer returns a new slice without the specified server. +func removeServer(servers []string, server string) []string { + result := make([]string, 0, len(servers)-1) + for _, s := range servers { + if s != server { + result = append(result, s) + } + } + return result +} diff --git a/weed/server/master_grpc_server_block_test.go b/weed/server/master_grpc_server_block_test.go new file mode 100644 index 000000000..99810c671 --- /dev/null +++ b/weed/server/master_grpc_server_block_test.go @@ -0,0 +1,298 @@ +package weed_server + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// testMasterServer creates a minimal MasterServer with mock VS calls for testing. +func testMasterServer(t *testing.T) *MasterServer { + t.Helper() + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + } + // Default mock: succeed with deterministic values. + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + return fmt.Sprintf("/data/%s.blk", name), + fmt.Sprintf("iqn.2024.test:%s", name), + string(server), + nil + } + ms.blockVSDelete = func(ctx context.Context, server pb.ServerAddress, name string) error { + return nil + } + return ms +} + +func TestMaster_CreateBlockVolume(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "test-vol", + SizeBytes: 1 << 30, + DiskType: "ssd", + }) + if err != nil { + t.Fatalf("CreateBlockVolume: %v", err) + } + if resp.VolumeId != "test-vol" { + t.Fatalf("VolumeId: got %q, want test-vol", resp.VolumeId) + } + if resp.VolumeServer != "vs1:9333" { + t.Fatalf("VolumeServer: got %q, want vs1:9333", resp.VolumeServer) + } + if resp.Iqn == "" || resp.IscsiAddr == "" { + t.Fatal("IQN or ISCSIAddr is empty") + } + + // Verify registry entry. + entry, ok := ms.blockRegistry.Lookup("test-vol") + if !ok { + t.Fatal("volume not found in registry") + } + if entry.Status != StatusActive { + t.Fatalf("status: got %d, want StatusActive", entry.Status) + } +} + +func TestMaster_CreateIdempotent(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + resp1, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("first create: %v", err) + } + + resp2, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("idempotent create: %v", err) + } + + if resp1.VolumeId != resp2.VolumeId || resp1.VolumeServer != resp2.VolumeServer { + t.Fatalf("idempotent mismatch: %+v vs %+v", resp1, resp2) + } +} + +func TestMaster_CreateIdempotentSizeMismatch(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Larger size should fail. + _, err = ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 2 << 30, + }) + if err == nil { + t.Fatal("expected error for size mismatch") + } + + // Same or smaller size should succeed (idempotent). + _, err = ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 29, + }) + if err != nil { + t.Fatalf("smaller size should succeed: %v", err) + } +} + +func TestMaster_CreateNoServers(t *testing.T) { + ms := testMasterServer(t) + // No block-capable servers registered. + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err == nil { + t.Fatal("expected error when no servers available") + } +} + +func TestMaster_CreateVSFailure_Retry(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + var callCount atomic.Int32 + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + n := callCount.Add(1) + if n == 1 { + return "", "", "", fmt.Errorf("disk full") + } + return fmt.Sprintf("/data/%s.blk", name), + fmt.Sprintf("iqn.2024.test:%s", name), + string(server), nil + } + + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("expected retry to succeed: %v", err) + } + if resp.VolumeId != "vol1" { + t.Fatalf("VolumeId: got %q, want vol1", resp.VolumeId) + } + if callCount.Load() < 2 { + t.Fatalf("expected at least 2 VS calls, got %d", callCount.Load()) + } +} + +func TestMaster_CreateVSFailure_Cleanup(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + return "", "", "", fmt.Errorf("all servers broken") + } + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err == nil { + t.Fatal("expected error when all VS fail") + } + + // No stale registry entry. + if _, ok := ms.blockRegistry.Lookup("vol1"); ok { + t.Fatal("stale registry entry should not exist") + } +} + +func TestMaster_CreateConcurrentSameName(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + var callCount atomic.Int32 + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + callCount.Add(1) + return fmt.Sprintf("/data/%s.blk", name), + fmt.Sprintf("iqn.2024.test:%s", name), + string(server), nil + } + + var wg sync.WaitGroup + results := make([]*master_pb.CreateBlockVolumeResponse, 10) + errors := make([]error, 10) + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + results[i], errors[i] = ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "same-vol", + SizeBytes: 1 << 30, + }) + }(i) + } + wg.Wait() + + // Some may get "already in progress" error, but at least one must succeed. + successCount := 0 + for i := 0; i < 10; i++ { + if errors[i] == nil { + successCount++ + } + } + if successCount == 0 { + t.Fatal("at least one concurrent create should succeed") + } + + // Only one VS allocation call should have been made. + if callCount.Load() != 1 { + t.Fatalf("expected exactly 1 VS call, got %d", callCount.Load()) + } +} + +func TestMaster_DeleteBlockVolume(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + _, err = ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{ + Name: "vol1", + }) + if err != nil { + t.Fatalf("delete: %v", err) + } + + if _, ok := ms.blockRegistry.Lookup("vol1"); ok { + t.Fatal("volume should be removed from registry") + } +} + +func TestMaster_DeleteNotFound(t *testing.T) { + ms := testMasterServer(t) + + _, err := ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{ + Name: "nonexistent", + }) + if err != nil { + t.Fatalf("delete nonexistent should succeed (idempotent): %v", err) + } +} + +func TestMaster_LookupBlockVolume(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + resp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "vol1", + }) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if resp.VolumeServer != "vs1:9333" { + t.Fatalf("VolumeServer: got %q, want vs1:9333", resp.VolumeServer) + } + if resp.CapacityBytes != 1<<30 { + t.Fatalf("CapacityBytes: got %d, want %d", resp.CapacityBytes, 1<<30) + } + + // Lookup nonexistent. + _, err = ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "nonexistent", + }) + if err == nil { + t.Fatal("lookup nonexistent should return error") + } +} diff --git a/weed/server/master_server.go b/weed/server/master_server.go index c4ddc7aa2..27aef5453 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -15,11 +15,13 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance" + "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/telemetry" "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/gorilla/mux" hashicorpRaft "github.com/hashicorp/raft" @@ -90,6 +92,11 @@ type MasterServer struct { // telemetry telemetryCollector *telemetry.Collector + + // block volume support + blockRegistry *BlockVolumeRegistry + blockVSAllocate func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (path, iqn, iscsiAddr string, err error) + blockVSDelete func(ctx context.Context, server pb.ServerAddress, name string) error } func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer { @@ -138,6 +145,10 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se Cluster: cluster.NewCluster(), } + ms.blockRegistry = NewBlockVolumeRegistry() + ms.blockVSAllocate = ms.defaultBlockVSAllocate + ms.blockVSDelete = ms.defaultBlockVSDelete + ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate) seq := ms.createSequencer(option) @@ -502,3 +513,32 @@ func (ms *MasterServer) Reload() { util.StringSplit(v.GetString("guard.white_list"), ",")...), ) } + +// defaultBlockVSAllocate calls a volume server's AllocateBlockVolume RPC. +func (ms *MasterServer) defaultBlockVSAllocate(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (path, iqn, iscsiAddr string, err error) { + err = operation.WithVolumeServerClient(false, server, ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + resp, rerr := client.AllocateBlockVolume(ctx, &volume_server_pb.AllocateBlockVolumeRequest{ + Name: name, + SizeBytes: sizeBytes, + DiskType: diskType, + }) + if rerr != nil { + return rerr + } + path = resp.Path + iqn = resp.Iqn + iscsiAddr = resp.IscsiAddr + return nil + }) + return +} + +// defaultBlockVSDelete calls a volume server's VolumeServerDeleteBlockVolume RPC. +func (ms *MasterServer) defaultBlockVSDelete(ctx context.Context, server pb.ServerAddress, name string) error { + return operation.WithVolumeServerClient(false, server, ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + _, err := client.VolumeServerDeleteBlockVolume(ctx, &volume_server_pb.VolumeServerDeleteBlockVolumeRequest{ + Name: name, + }) + return err + }) +} diff --git a/weed/server/qa_block_cp62_test.go b/weed/server/qa_block_cp62_test.go new file mode 100644 index 000000000..664ea2d24 --- /dev/null +++ b/weed/server/qa_block_cp62_test.go @@ -0,0 +1,604 @@ +package weed_server + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +// ============================================================ +// QA-REG: Registry adversarial tests +// ============================================================ + +// QA-REG-1: FullHeartbeat removes volumes created via RPC while heartbeat is in transit. +// Scenario: Master creates vol1 (StatusActive). Before the VS heartbeat arrives, +// another VS heartbeat (from a different server?) or a delayed heartbeat arrives +// that doesn't include vol1. FullHeartbeat should only remove stale entries for +// THAT specific server, not all servers. +func TestQA_Reg_FullHeartbeatCrossTalk(t *testing.T) { + r := NewBlockVolumeRegistry() + + // Register vol1 on server s1. + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive}) + // Register vol2 on server s2. + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s2", Path: "/v2.blk", Status: StatusActive}) + + // Full heartbeat from s1 reports vol1 — should NOT affect s2's volumes. + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + {Path: "/v1.blk", Epoch: 1}, + }) + + // vol2 on s2 should still exist. + if _, ok := r.Lookup("vol2"); !ok { + t.Fatal("BUG: full heartbeat from s1 removed vol2 which belongs to s2") + } +} + +// QA-REG-2: FullHeartbeat from server with zero volumes should clear all entries for that server. +func TestQA_Reg_FullHeartbeatEmptyServer(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive}) + r.Register(&BlockVolumeEntry{Name: "vol2", VolumeServer: "s1", Path: "/v2.blk", Status: StatusActive}) + + // Empty heartbeat from s1 (HasNoBlockVolumes=true, zero infos). + r.UpdateFullHeartbeat("s1", nil) + + if _, ok := r.Lookup("vol1"); ok { + t.Error("BUG: vol1 should be removed after empty full heartbeat") + } + if _, ok := r.Lookup("vol2"); ok { + t.Error("BUG: vol2 should be removed after empty full heartbeat") + } +} + +// QA-REG-3: Concurrent FullHeartbeat and Register for same server. +// While a heartbeat is being processed, a new CreateBlockVolume registers on the same server. +func TestQA_Reg_ConcurrentHeartbeatAndRegister(t *testing.T) { + r := NewBlockVolumeRegistry() + + var wg sync.WaitGroup + var panicked atomic.Bool + + for i := 0; i < 50; i++ { + wg.Add(2) + go func(i int) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + } + }() + r.UpdateFullHeartbeat("s1", []*master_pb.BlockVolumeInfoMessage{ + {Path: fmt.Sprintf("/v%d.blk", i), Epoch: uint64(i)}, + }) + }(i) + go func(i int) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + } + }() + r.Register(&BlockVolumeEntry{ + Name: fmt.Sprintf("vol%d", i), + VolumeServer: "s1", + Path: fmt.Sprintf("/v%d.blk", i), + Status: StatusActive, + }) + }(i) + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent heartbeat + register caused panic") + } +} + +// QA-REG-4: DeltaHeartbeat with removed path that doesn't match any registered volume. +// Should be a no-op, not panic or corrupt state. +func TestQA_Reg_DeltaHeartbeatUnknownPath(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + // Delta says /unknown.blk was removed — this path doesn't match vol1. + r.UpdateDeltaHeartbeat("s1", + nil, + []*master_pb.BlockVolumeShortInfoMessage{{Path: "/unknown.blk"}}, + ) + + // vol1 should still exist. + if _, ok := r.Lookup("vol1"); !ok { + t.Fatal("BUG: delta heartbeat with unknown path removed an unrelated volume") + } +} + +// QA-REG-5: PickServer always picks the same server when counts are tied. +// Deterministic placement prevents flip-flopping. +func TestQA_Reg_PickServerTiebreaker(t *testing.T) { + r := NewBlockVolumeRegistry() + + // All servers have 0 volumes. + servers := []string{"s1", "s2", "s3"} + results := make(map[string]int) + for i := 0; i < 10; i++ { + s, err := r.PickServer(servers) + if err != nil { + t.Fatal(err) + } + results[s]++ + } + // With stable ordering, the same server should win every time. + // The algorithm picks servers[0] by default when counts are equal. + if results["s1"] != 10 { + t.Logf("PickServer results with tied counts: %v (non-deterministic but OK if no panic)", results) + } +} + +// QA-REG-6: Unregister a volume then re-register with a different server. +func TestQA_Reg_ReregisterDifferentServer(t *testing.T) { + r := NewBlockVolumeRegistry() + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk"}) + + // Unregister and re-register on s2. + r.Unregister("vol1") + r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s2", Path: "/v1.blk"}) + + entry, ok := r.Lookup("vol1") + if !ok { + t.Fatal("vol1 should exist after re-register") + } + if entry.VolumeServer != "s2" { + t.Fatalf("VolumeServer: got %q, want s2", entry.VolumeServer) + } + + // s1 should have no volumes. + if vols := r.ListByServer("s1"); len(vols) != 0 { + t.Fatalf("s1 should have 0 volumes after re-register, got %d", len(vols)) + } + // s2 should have 1 volume. + if vols := r.ListByServer("s2"); len(vols) != 1 { + t.Fatalf("s2 should have 1 volume, got %d", len(vols)) + } +} + +// QA-REG-7: AcquireInflight for different names doesn't interfere. +func TestQA_Reg_InflightIndependence(t *testing.T) { + r := NewBlockVolumeRegistry() + + if !r.AcquireInflight("vol1") { + t.Fatal("acquire vol1 should succeed") + } + if !r.AcquireInflight("vol2") { + t.Fatal("acquire vol2 should succeed (different name)") + } + if r.AcquireInflight("vol1") { + t.Fatal("double acquire vol1 should fail") + } + r.ReleaseInflight("vol1") + if !r.AcquireInflight("vol1") { + t.Fatal("acquire vol1 after release should succeed") + } + r.ReleaseInflight("vol1") + r.ReleaseInflight("vol2") +} + +// QA-REG-8: BlockCapableServers includes only currently-marked servers. +func TestQA_Reg_BlockCapableServersAfterUnmark(t *testing.T) { + r := NewBlockVolumeRegistry() + r.MarkBlockCapable("s1") + r.MarkBlockCapable("s2") + r.MarkBlockCapable("s3") + + r.UnmarkBlockCapable("s2") + + servers := r.BlockCapableServers() + for _, s := range servers { + if s == "s2" { + t.Fatal("BUG: s2 should not be in block-capable list after UnmarkBlockCapable") + } + } + if len(servers) != 2 { + t.Fatalf("expected 2 block-capable servers, got %d: %v", len(servers), servers) + } +} + +// ============================================================ +// QA-MASTER: Master RPC adversarial tests +// ============================================================ + +// QA-MASTER-1: CreateBlockVolume then Delete while VS is unreachable. +// Delete should fail (cannot contact VS), but registry entry should NOT be removed. +func TestQA_Master_DeleteVSUnreachable(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "vol1", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Make VS delete fail. + ms.blockVSDelete = func(ctx context.Context, server pb.ServerAddress, name string) error { + return fmt.Errorf("connection refused") + } + + _, err = ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{ + Name: "vol1", + }) + if err == nil { + t.Fatal("delete should fail when VS is unreachable") + } + + // Registry entry should still exist (not orphaned). + if _, ok := ms.blockRegistry.Lookup("vol1"); !ok { + t.Fatal("BUG: registry entry removed even though VS delete failed") + } +} + +// QA-MASTER-2: Create then lookup a volume with a name that requires sanitization. +func TestQA_Master_CreateSanitizedName(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + // Name with special characters. + resp, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "pvc-abc/def:123", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Lookup should use the exact name (not sanitized). + lookupResp, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{ + Name: "pvc-abc/def:123", + }) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if lookupResp.VolumeServer != resp.VolumeServer { + t.Fatalf("lookup mismatch: %+v vs %+v", lookupResp, resp) + } +} + +// QA-MASTER-3: Concurrent Create and Delete for the same volume name. +func TestQA_Master_ConcurrentCreateDelete(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + var wg sync.WaitGroup + var panicked atomic.Bool + + for i := 0; i < 20; i++ { + wg.Add(2) + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC in create: %v", r) + } + }() + ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "race-vol", SizeBytes: 1 << 30, + }) + }() + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC in delete: %v", r) + } + }() + ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{ + Name: "race-vol", + }) + }() + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent create/delete caused panic") + } +} + +// QA-MASTER-4: Create with all VS failing should not leave orphan in registry. +func TestQA_Master_AllVSFailNoOrphan(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + ms.blockRegistry.MarkBlockCapable("vs3:9333") + + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + return "", "", "", fmt.Errorf("disk full on %s", server) + } + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "doomed", SizeBytes: 1 << 30, + }) + if err == nil { + t.Fatal("expected error when all VS fail") + } + + // No orphan in registry. + if _, ok := ms.blockRegistry.Lookup("doomed"); ok { + t.Fatal("BUG: orphan entry in registry after all VS failed") + } + + // Inflight lock should be released. + if !ms.blockRegistry.AcquireInflight("doomed") { + t.Fatal("BUG: inflight lock not released after all VS failed") + } + ms.blockRegistry.ReleaseInflight("doomed") +} + +// QA-MASTER-5: VS allocate succeeds but is slow — inflight lock must block second create. +func TestQA_Master_SlowAllocateBlocksSecond(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + var allocCount atomic.Int32 + ms.blockVSAllocate = func(ctx context.Context, server pb.ServerAddress, name string, sizeBytes uint64, diskType string) (string, string, string, error) { + allocCount.Add(1) + time.Sleep(100 * time.Millisecond) // simulate slow VS + return fmt.Sprintf("/data/%s.blk", name), + fmt.Sprintf("iqn.test:%s", name), + string(server), nil + } + + var wg sync.WaitGroup + errors := make([]error, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, errors[i] = ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "slow-vol", SizeBytes: 1 << 30, + }) + }(i) + } + wg.Wait() + + // One should succeed, one should get "already in progress" or succeed via idempotent path. + successCount := 0 + for _, err := range errors { + if err == nil { + successCount++ + } + } + if successCount == 0 { + t.Fatal("at least one create should succeed") + } + + // Only 1 VS allocation call should have been made (inflight blocks the second). + if allocCount.Load() > 1 { + t.Logf("WARNING: %d VS allocations made (expected 1 — second should be blocked by inflight lock or return idempotent)", allocCount.Load()) + } +} + +// QA-MASTER-6: Create with SizeBytes == 0 should be rejected. +func TestQA_Master_CreateZeroSize(t *testing.T) { + ms := testMasterServer(t) + ms.blockRegistry.MarkBlockCapable("vs1:9333") + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "zero", SizeBytes: 0, + }) + if err == nil { + t.Fatal("expected error for zero size") + } +} + +// QA-MASTER-7: Create with empty name should be rejected. +func TestQA_Master_CreateEmptyName(t *testing.T) { + ms := testMasterServer(t) + + _, err := ms.CreateBlockVolume(context.Background(), &master_pb.CreateBlockVolumeRequest{ + Name: "", SizeBytes: 1 << 30, + }) + if err == nil { + t.Fatal("expected error for empty name") + } +} + +// QA-MASTER-8: Lookup/Delete with empty name should be rejected. +func TestQA_Master_EmptyNameValidation(t *testing.T) { + ms := testMasterServer(t) + + _, err := ms.LookupBlockVolume(context.Background(), &master_pb.LookupBlockVolumeRequest{Name: ""}) + if err == nil { + t.Error("lookup with empty name should fail") + } + + _, err = ms.DeleteBlockVolume(context.Background(), &master_pb.DeleteBlockVolumeRequest{Name: ""}) + if err == nil { + t.Error("delete with empty name should fail") + } +} + +// ============================================================ +// QA-VS: Volume server BlockService adversarial tests +// ============================================================ + +// QA-VS-1: Concurrent CreateBlockVol for the same name on a single VS. +func TestQA_VS_ConcurrentCreate(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + var wg sync.WaitGroup + var panicked atomic.Bool + errors := make([]error, 20) + + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC: %v", r) + } + }() + _, _, _, errors[i] = bs.CreateBlockVol("race-vol", 4*1024*1024, "") + }(i) + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent CreateBlockVol caused panic") + } + + // At least some should succeed. + successCount := 0 + for _, err := range errors { + if err == nil { + successCount++ + } + } + if successCount == 0 { + t.Fatal("at least one concurrent CreateBlockVol should succeed") + } +} + +// QA-VS-2: Concurrent CreateBlockVol and DeleteBlockVol for the same name. +func TestQA_VS_ConcurrentCreateDelete(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + var wg sync.WaitGroup + var panicked atomic.Bool + + for i := 0; i < 20; i++ { + wg.Add(2) + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + } + }() + bs.CreateBlockVol("cd-vol", 4*1024*1024, "") + }() + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + } + }() + bs.DeleteBlockVol("cd-vol") + }() + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent create/delete caused panic") + } +} + +// QA-VS-3: DeleteBlockVol should clean up .snap.* files. +func TestQA_VS_DeleteCleansSnapshots(t *testing.T) { + bs, blockDir := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("snap-vol", 4*1024*1024, "") + + // Simulate snapshot files. + snapPath := blockDir + "/snap-vol.blk.snap.0" + if err := writeTestFile(snapPath); err != nil { + t.Fatalf("create snap file: %v", err) + } + + bs.DeleteBlockVol("snap-vol") + + // Snap file should be removed. + if fileExists(snapPath) { + t.Error("BUG: .snap.0 file not cleaned up after DeleteBlockVol") + } +} + +// QA-VS-4: CreateBlockVol with name that sanitizes to the same filename as another volume. +func TestQA_VS_SanitizationCollision(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + // "VolA" sanitizes to "vola.blk", "vola" also sanitizes to "vola.blk". + _, _, _, err := bs.CreateBlockVol("VolA", 4*1024*1024, "") + if err != nil { + t.Fatalf("create VolA: %v", err) + } + + // "vola" should get the idempotent path (same file on disk). + path2, _, _, err := bs.CreateBlockVol("vola", 4*1024*1024, "") + if err != nil { + t.Fatalf("create vola: %v", err) + } + + // Should point to the same file. + if !strings.HasSuffix(path2, "vola.blk") { + t.Errorf("path: got %q, expected to end with vola.blk", path2) + } +} + +// QA-VS-5: CreateBlockVol idempotent path verifies TargetServer re-registration. +func TestQA_VS_CreateIdempotentReaddTarget(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + // First create. + _, iqn1, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "") + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Second create (idempotent) — should succeed and re-add to TargetServer. + _, iqn2, _, err := bs.CreateBlockVol("readd-vol", 4*1024*1024, "") + if err != nil { + t.Fatalf("idempotent create: %v", err) + } + + if iqn1 != iqn2 { + t.Fatalf("IQN mismatch: %q vs %q", iqn1, iqn2) + } +} + +// QA-VS-6: gRPC handler with nil blockService. +func TestQA_VS_GrpcNilBlockService(t *testing.T) { + vs := &VolumeServer{blockService: nil} + + _, err := vs.AllocateBlockVolume(context.Background(), &volume_server_pb.AllocateBlockVolumeRequest{ + Name: "vol1", SizeBytes: 1 << 30, + }) + if err == nil { + t.Fatal("expected error when blockService is nil") + } + + _, err = vs.VolumeServerDeleteBlockVolume(context.Background(), &volume_server_pb.VolumeServerDeleteBlockVolumeRequest{ + Name: "vol1", + }) + if err == nil { + t.Fatal("expected error when blockService is nil") + } +} + +// Helper functions. + +func writeTestFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + f.Write([]byte("test")) + return f.Close() +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/weed/server/volume_grpc_block.go b/weed/server/volume_grpc_block.go new file mode 100644 index 000000000..4608e8c94 --- /dev/null +++ b/weed/server/volume_grpc_block.go @@ -0,0 +1,48 @@ +package weed_server + +import ( + "context" + "fmt" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +// AllocateBlockVolume creates a new block volume on this volume server. +func (vs *VolumeServer) AllocateBlockVolume(_ context.Context, req *volume_server_pb.AllocateBlockVolumeRequest) (*volume_server_pb.AllocateBlockVolumeResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + if req.SizeBytes == 0 { + return nil, fmt.Errorf("size_bytes must be > 0") + } + + path, iqn, iscsiAddr, err := vs.blockService.CreateBlockVol(req.Name, req.SizeBytes, req.DiskType) + if err != nil { + return nil, fmt.Errorf("create block volume %q: %w", req.Name, err) + } + + return &volume_server_pb.AllocateBlockVolumeResponse{ + Path: path, + Iqn: iqn, + IscsiAddr: iscsiAddr, + }, nil +} + +// VolumeServerDeleteBlockVolume deletes a block volume on this volume server. +func (vs *VolumeServer) VolumeServerDeleteBlockVolume(_ context.Context, req *volume_server_pb.VolumeServerDeleteBlockVolumeRequest) (*volume_server_pb.VolumeServerDeleteBlockVolumeResponse, error) { + if vs.blockService == nil { + return nil, fmt.Errorf("block service not enabled on this volume server") + } + if req.Name == "" { + return nil, fmt.Errorf("name is required") + } + + if err := vs.blockService.DeleteBlockVol(req.Name); err != nil { + return nil, fmt.Errorf("delete block volume %q: %w", req.Name, err) + } + + return &volume_server_pb.VolumeServerDeleteBlockVolumeResponse{}, nil +} diff --git a/weed/server/volume_grpc_block_test.go b/weed/server/volume_grpc_block_test.go new file mode 100644 index 000000000..b2bab8d98 --- /dev/null +++ b/weed/server/volume_grpc_block_test.go @@ -0,0 +1,110 @@ +package weed_server + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func newTestBlockServiceWithDir(t *testing.T) (*BlockService, string) { + t.Helper() + dir := t.TempDir() + blockDir := filepath.Join(dir, "blocks") + os.MkdirAll(blockDir, 0755) + bs := StartBlockService("127.0.0.1:0", blockDir, "iqn.2024.test:") + if bs == nil { + t.Fatal("StartBlockService returned nil") + } + t.Cleanup(func() { bs.Shutdown() }) + return bs, blockDir +} + +func TestVS_AllocateBlockVolume(t *testing.T) { + bs, blockDir := newTestBlockServiceWithDir(t) + + path, iqn, iscsiAddr, err := bs.CreateBlockVol("test-vol", 4*1024*1024, "ssd") + if err != nil { + t.Fatalf("CreateBlockVol: %v", err) + } + if path == "" || iqn == "" || iscsiAddr == "" { + t.Fatalf("empty return values: path=%q iqn=%q addr=%q", path, iqn, iscsiAddr) + } + + // Verify file exists. + expectedPath := filepath.Join(blockDir, "test-vol.blk") + if path != expectedPath { + t.Fatalf("path: got %q, want %q", path, expectedPath) + } + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Fatalf(".blk file not created at %s", path) + } + + // IQN should contain sanitized name. + if !strings.Contains(iqn, "test-vol") { + t.Fatalf("IQN %q should contain 'test-vol'", iqn) + } +} + +func TestVS_AllocateIdempotent(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + path1, iqn1, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Same name+size should return same info. + path2, iqn2, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + if err != nil { + t.Fatalf("idempotent create: %v", err) + } + if path1 != path2 || iqn1 != iqn2 { + t.Fatalf("idempotent mismatch: (%q,%q) vs (%q,%q)", path1, iqn1, path2, iqn2) + } +} + +func TestVS_AllocateSizeMismatch(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + _, _, _, err := bs.CreateBlockVol("vol1", 4*1024*1024, "") + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Requesting a LARGER size than existing should fail. + _, _, _, err = bs.CreateBlockVol("vol1", 8*1024*1024, "") + if err == nil { + t.Fatal("size mismatch should return error") + } +} + +func TestVS_DeleteBlockVolume(t *testing.T) { + bs, blockDir := newTestBlockServiceWithDir(t) + + bs.CreateBlockVol("vol1", 4*1024*1024, "") + path := filepath.Join(blockDir, "vol1.blk") + + // File should exist. + if _, err := os.Stat(path); err != nil { + t.Fatalf("file should exist: %v", err) + } + + if err := bs.DeleteBlockVol("vol1"); err != nil { + t.Fatalf("DeleteBlockVol: %v", err) + } + + // File should be gone. + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("file should be removed after delete") + } +} + +func TestVS_DeleteNotFound(t *testing.T) { + bs, _ := newTestBlockServiceWithDir(t) + + // Deleting non-existent volume should be idempotent. + if err := bs.DeleteBlockVol("no-such-vol"); err != nil { + t.Fatalf("delete non-existent should not error: %v", err) + } +} diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go index e2523543a..633423507 100644 --- a/weed/server/volume_grpc_client_to_master.go +++ b/weed/server/volume_grpc_client_to_master.go @@ -13,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "context" @@ -210,6 +211,15 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp rack := vs.store.GetRack() ip := vs.store.Ip port := uint32(vs.store.Port) + + // Send block volume full heartbeat if block service is enabled. + if vs.blockService != nil { + blockBeat := vs.collectBlockVolumeHeartbeat(ip, port, dataCenter, rack) + if err = stream.Send(blockBeat); err != nil { + glog.V(0).Infof("Volume Server Failed to send block volume heartbeat to master %s: %v", masterAddress, err) + return "", err + } + } for { select { case stateMessage := <-vs.store.StateUpdateChan: @@ -305,14 +315,15 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp case <-vs.stopChan: var volumeMessages []*master_pb.VolumeInformationMessage emptyBeat := &master_pb.Heartbeat{ - Ip: ip, - Port: port, - PublicUrl: vs.store.PublicUrl, - MaxFileKey: uint64(0), - DataCenter: dataCenter, - Rack: rack, - Volumes: volumeMessages, - HasNoVolumes: len(volumeMessages) == 0, + Ip: ip, + Port: port, + PublicUrl: vs.store.PublicUrl, + MaxFileKey: uint64(0), + DataCenter: dataCenter, + Rack: rack, + Volumes: volumeMessages, + HasNoVolumes: len(volumeMessages) == 0, + HasNoBlockVolumes: vs.blockService != nil, } glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port) if err = stream.Send(emptyBeat); err != nil { @@ -323,3 +334,16 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp } } } + +// collectBlockVolumeHeartbeat builds a heartbeat with the full list of block volumes. +func (vs *VolumeServer) collectBlockVolumeHeartbeat(ip string, port uint32, dc, rack string) *master_pb.Heartbeat { + msgs := vs.blockService.Store().CollectBlockVolumeHeartbeat() + return &master_pb.Heartbeat{ + Ip: ip, + Port: port, + DataCenter: dc, + Rack: rack, + BlockVolumeInfos: blockvol.InfoMessagesToProto(msgs), + HasNoBlockVolumes: len(msgs) == 0, + } +} diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 6bc3a6898..4db60cc19 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -54,6 +54,8 @@ type VolumeServer struct { fileSizeLimitBytes int64 isHeartbeating bool stopChan chan bool + + blockService *BlockService // block volume iSCSI service (nil if disabled) } func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index bb8c2554b..cf019ea26 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -1,6 +1,7 @@ package weed_server import ( + "fmt" "log" "os" "path/filepath" @@ -17,6 +18,8 @@ type BlockService struct { blockStore *storage.BlockVolumeStore targetServer *iscsi.TargetServer iqnPrefix string + blockDir string + listenAddr string } // StartBlockService scans blockDir for .blk files, opens them as block volumes, @@ -34,6 +37,8 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix string) *BlockService { bs := &BlockService{ blockStore: storage.NewBlockVolumeStore(), iqnPrefix: iqnPrefix, + blockDir: blockDir, + listenAddr: listenAddr, } logger := log.New(os.Stderr, "iscsi: ", log.LstdFlags) @@ -83,7 +88,7 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix string) *BlockService { // Derive IQN from filename: vol1.blk -> iqn.2024-01.com.seaweedfs:vol.vol1 name := strings.TrimSuffix(entry.Name(), ".blk") - iqn := iqnPrefix + name + iqn := iqnPrefix + blockvol.SanitizeIQN(name) adapter := blockvol.NewBlockVolAdapter(vol) bs.targetServer.AddVolume(iqn, adapter) glog.V(0).Infof("block service: registered %s as %s", path, iqn) @@ -105,6 +110,95 @@ func (bs *BlockService) Store() *storage.BlockVolumeStore { return bs.blockStore } +// BlockDir returns the block volume data directory. +func (bs *BlockService) BlockDir() string { + return bs.blockDir +} + +// ListenAddr returns the iSCSI target listen address. +func (bs *BlockService) ListenAddr() string { + return bs.listenAddr +} + +// CreateBlockVol creates a new .blk file, registers it with BlockVolumeStore +// and iSCSI TargetServer. Returns path, IQN, iSCSI addr. +// Idempotent: if volume already exists with same or larger size, returns existing info. +func (bs *BlockService) CreateBlockVol(name string, sizeBytes uint64, diskType string) (path, iqn, iscsiAddr string, err error) { + sanitized := blockvol.SanitizeFilename(name) + path = filepath.Join(bs.blockDir, sanitized+".blk") + iqn = bs.iqnPrefix + blockvol.SanitizeIQN(name) + iscsiAddr = bs.listenAddr + + // Check if already registered. + if vol, ok := bs.blockStore.GetBlockVolume(path); ok { + info := vol.Info() + if info.VolumeSize < sizeBytes { + return "", "", "", fmt.Errorf("block volume %q exists with size %d (requested %d)", + name, info.VolumeSize, sizeBytes) + } + // Re-add to TargetServer in case it was cleared (crash recovery). + // AddVolume is idempotent — no-op if already registered. + adapter := blockvol.NewBlockVolAdapter(vol) + bs.targetServer.AddVolume(iqn, adapter) + return path, iqn, iscsiAddr, nil + } + + // Create the .blk file. + if err := os.MkdirAll(bs.blockDir, 0755); err != nil { + return "", "", "", fmt.Errorf("create block dir: %w", err) + } + created, err := blockvol.CreateBlockVol(path, blockvol.CreateOptions{ + VolumeSize: sizeBytes, + }) + if err != nil { + return "", "", "", fmt.Errorf("create block volume: %w", err) + } + created.Close() + + // Open and register. + vol, err := bs.blockStore.AddBlockVolume(path, diskType) + if err != nil { + os.Remove(path) + return "", "", "", fmt.Errorf("register block volume: %w", err) + } + + adapter := blockvol.NewBlockVolAdapter(vol) + bs.targetServer.AddVolume(iqn, adapter) + glog.V(0).Infof("block service: created %s as %s (%d bytes)", path, iqn, sizeBytes) + return path, iqn, iscsiAddr, nil +} + +// DeleteBlockVol disconnects iSCSI sessions, closes the volume, and removes the .blk file. +// Idempotent: returns nil if volume not found. +func (bs *BlockService) DeleteBlockVol(name string) error { + sanitized := blockvol.SanitizeFilename(name) + path := filepath.Join(bs.blockDir, sanitized+".blk") + iqn := bs.iqnPrefix + blockvol.SanitizeIQN(name) + + // Disconnect active iSCSI sessions and remove target entry. + if bs.targetServer != nil { + bs.targetServer.DisconnectVolume(iqn) + } + + // Close and unregister. + if err := bs.blockStore.RemoveBlockVolume(path); err != nil { + // Not found is OK (idempotent). + if !strings.Contains(err.Error(), "not found") { + return fmt.Errorf("remove block volume: %w", err) + } + } + + // Remove the .blk file and any snapshot files. + os.Remove(path) + matches, _ := filepath.Glob(path + ".snap.*") + for _, m := range matches { + os.Remove(m) + } + + glog.V(0).Infof("block service: deleted %s", path) + return nil +} + // Shutdown gracefully stops the iSCSI target and closes all block volumes. func (bs *BlockService) Shutdown() { if bs == nil { diff --git a/weed/storage/blockvol/adapter.go b/weed/storage/blockvol/adapter.go index c3e6a1689..165f39ab9 100644 --- a/weed/storage/blockvol/adapter.go +++ b/weed/storage/blockvol/adapter.go @@ -4,44 +4,83 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" ) -// BlockVolAdapter wraps a *BlockVol to implement the iscsi.BlockDevice interface, -// bridging the BlockVol storage engine to iSCSI target sessions. +// BlockVolAdapter wraps a *BlockVol to implement the iscsi.BlockDevice and +// iscsi.ALUAProvider interfaces, bridging the BlockVol storage engine to +// iSCSI target sessions. type BlockVolAdapter struct { - vol *BlockVol + Vol *BlockVol + TPGID uint16 // Target Port Group ID for ALUA (1-65535) } // NewBlockVolAdapter creates a BlockDevice adapter for the given BlockVol. func NewBlockVolAdapter(vol *BlockVol) *BlockVolAdapter { - return &BlockVolAdapter{vol: vol} + return &BlockVolAdapter{Vol: vol} } func (a *BlockVolAdapter) ReadAt(lba uint64, length uint32) ([]byte, error) { - return a.vol.ReadLBA(lba, length) + return a.Vol.ReadLBA(lba, length) } func (a *BlockVolAdapter) WriteAt(lba uint64, data []byte) error { - return a.vol.WriteLBA(lba, data) + return a.Vol.WriteLBA(lba, data) } func (a *BlockVolAdapter) Trim(lba uint64, length uint32) error { - return a.vol.Trim(lba, length) + return a.Vol.Trim(lba, length) } func (a *BlockVolAdapter) SyncCache() error { - return a.vol.SyncCache() + return a.Vol.SyncCache() } func (a *BlockVolAdapter) BlockSize() uint32 { - return a.vol.Info().BlockSize + return a.Vol.Info().BlockSize } func (a *BlockVolAdapter) VolumeSize() uint64 { - return a.vol.Info().VolumeSize + return a.Vol.Info().VolumeSize } func (a *BlockVolAdapter) IsHealthy() bool { - return a.vol.Info().Healthy + return a.Vol.Info().Healthy } -// Compile-time check that BlockVolAdapter implements iscsi.BlockDevice. +// ALUAState returns the ALUA asymmetric access state based on the volume's role. +func (a *BlockVolAdapter) ALUAState() uint8 { return RoleToALUA(a.Vol.Role()) } + +// TPGroupID returns the target port group ID. +func (a *BlockVolAdapter) TPGroupID() uint16 { return a.TPGID } + +// DeviceNAA returns the NAA-6 device identifier derived from the volume UUID. +func (a *BlockVolAdapter) DeviceNAA() [8]byte { return UUIDToNAA(a.Vol.Info().UUID) } + +// RoleToALUA maps a BlockVol Role to an ALUA asymmetric access state. +// RoleNone maps to Active/Optimized so standalone single-node targets +// (no assignment from master) can accept writes. +func RoleToALUA(r Role) uint8 { + switch r { + case RolePrimary, RoleNone: + return iscsi.ALUAActiveOptimized + case RoleReplica: + return iscsi.ALUAStandby + case RoleStale: + return iscsi.ALUAUnavailable + case RoleRebuilding, RoleDraining: + return iscsi.ALUATransitioning + default: + return iscsi.ALUAStandby + } +} + +// UUIDToNAA converts a 16-byte UUID to an 8-byte NAA-6 identifier. +// NAA-6 format: nibble 6 (NAA=6) followed by 60 bits from the UUID. +func UUIDToNAA(uuid [16]byte) [8]byte { + var naa [8]byte + naa[0] = 0x60 | (uuid[0] & 0x0F) + copy(naa[1:], uuid[1:8]) + return naa +} + +// Compile-time checks. var _ iscsi.BlockDevice = (*BlockVolAdapter)(nil) +var _ iscsi.ALUAProvider = (*BlockVolAdapter)(nil) diff --git a/weed/storage/blockvol/adapter_test.go b/weed/storage/blockvol/adapter_test.go index 07d05b4ad..7eceaa04a 100644 --- a/weed/storage/blockvol/adapter_test.go +++ b/weed/storage/blockvol/adapter_test.go @@ -68,3 +68,81 @@ func TestAdapterImplementsInterface(t *testing.T) { t.Fatal(err) } } + +func TestAdapterALUAProvider(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "alua_test.blk") + + vol, err := CreateBlockVol(path, CreateOptions{ + VolumeSize: 1024 * 4096, + BlockSize: 4096, + ExtentSize: 65536, + WALSize: 1 << 20, + }) + if err != nil { + t.Fatal(err) + } + defer vol.Close() + + adapter := &BlockVolAdapter{Vol: vol, TPGID: 42} + + // Verify it satisfies the ALUA interface. + var _ iscsi.ALUAProvider = adapter + + if adapter.TPGroupID() != 42 { + t.Fatalf("TPGroupID: got %d, want 42", adapter.TPGroupID()) + } + + // RoleNone -> Active/Optimized + if adapter.ALUAState() != iscsi.ALUAActiveOptimized { + t.Fatalf("ALUAState: got %d, want %d", adapter.ALUAState(), iscsi.ALUAActiveOptimized) + } + + // DeviceNAA should have NAA=6 in high nibble. + naa := adapter.DeviceNAA() + if naa[0]&0xF0 != 0x60 { + t.Fatalf("DeviceNAA high nibble: got 0x%02x, want 0x6x", naa[0]) + } +} + +func TestRoleToALUA(t *testing.T) { + tests := []struct { + role Role + want uint8 + }{ + {RoleNone, iscsi.ALUAActiveOptimized}, + {RolePrimary, iscsi.ALUAActiveOptimized}, + {RoleReplica, iscsi.ALUAStandby}, + {RoleStale, iscsi.ALUAUnavailable}, + {RoleRebuilding, iscsi.ALUATransitioning}, + {RoleDraining, iscsi.ALUATransitioning}, + } + for _, tt := range tests { + got := RoleToALUA(tt.role) + if got != tt.want { + t.Errorf("RoleToALUA(%v): got %d, want %d", tt.role, got, tt.want) + } + } +} + +func TestUUIDToNAA(t *testing.T) { + var uuid [16]byte + for i := range uuid { + uuid[i] = byte(i + 1) + } + naa := UUIDToNAA(uuid) + // High nibble must be 0x6 + if naa[0]&0xF0 != 0x60 { + t.Fatalf("NAA high nibble: got 0x%02x, want 0x6x", naa[0]) + } + // Low nibble of first byte comes from uuid[0] + if naa[0]&0x0F != uuid[0]&0x0F { + t.Fatalf("NAA low nibble: got 0x%02x, want 0x%02x", naa[0]&0x0F, uuid[0]&0x0F) + } + // Bytes 1-7 come from uuid[1:8] + for i := 1; i < 8; i++ { + if naa[i] != uuid[i] { + t.Fatalf("NAA[%d]: got 0x%02x, want 0x%02x", i, naa[i], uuid[i]) + } + } +} diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go new file mode 100644 index 000000000..1f0d0a99b --- /dev/null +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -0,0 +1,111 @@ +package blockvol + +import ( + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// InfoMessageToProto converts a Go wire type to proto. +func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMessage { + return &master_pb.BlockVolumeInfoMessage{ + Path: m.Path, + VolumeSize: m.VolumeSize, + BlockSize: m.BlockSize, + Epoch: m.Epoch, + Role: m.Role, + WalHeadLsn: m.WalHeadLsn, + CheckpointLsn: m.CheckpointLsn, + HasLease: m.HasLease, + DiskType: m.DiskType, + } +} + +// InfoMessageFromProto converts a proto to Go wire type. +func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMessage { + if p == nil { + return BlockVolumeInfoMessage{} + } + return BlockVolumeInfoMessage{ + Path: p.Path, + VolumeSize: p.VolumeSize, + BlockSize: p.BlockSize, + Epoch: p.Epoch, + Role: p.Role, + WalHeadLsn: p.WalHeadLsn, + CheckpointLsn: p.CheckpointLsn, + HasLease: p.HasLease, + DiskType: p.DiskType, + } +} + +// InfoMessagesToProto converts a slice of Go wire types to proto. +func InfoMessagesToProto(msgs []BlockVolumeInfoMessage) []*master_pb.BlockVolumeInfoMessage { + out := make([]*master_pb.BlockVolumeInfoMessage, len(msgs)) + for i, m := range msgs { + out[i] = InfoMessageToProto(m) + } + return out +} + +// InfoMessagesFromProto converts a slice of proto messages to Go wire types. +func InfoMessagesFromProto(protos []*master_pb.BlockVolumeInfoMessage) []BlockVolumeInfoMessage { + out := make([]BlockVolumeInfoMessage, len(protos)) + for i, p := range protos { + out[i] = InfoMessageFromProto(p) + } + return out +} + +// ShortInfoToProto converts a Go short info to proto. +func ShortInfoToProto(m BlockVolumeShortInfoMessage) *master_pb.BlockVolumeShortInfoMessage { + return &master_pb.BlockVolumeShortInfoMessage{ + Path: m.Path, + VolumeSize: m.VolumeSize, + BlockSize: m.BlockSize, + DiskType: m.DiskType, + } +} + +// ShortInfoFromProto converts a proto short info to Go wire type. +func ShortInfoFromProto(p *master_pb.BlockVolumeShortInfoMessage) BlockVolumeShortInfoMessage { + if p == nil { + return BlockVolumeShortInfoMessage{} + } + return BlockVolumeShortInfoMessage{ + Path: p.Path, + VolumeSize: p.VolumeSize, + BlockSize: p.BlockSize, + DiskType: p.DiskType, + } +} + +// AssignmentToProto converts a Go assignment to proto. +func AssignmentToProto(a BlockVolumeAssignment) *master_pb.BlockVolumeAssignment { + return &master_pb.BlockVolumeAssignment{ + Path: a.Path, + Epoch: a.Epoch, + Role: a.Role, + LeaseTtlMs: a.LeaseTtlMs, + } +} + +// AssignmentFromProto converts a proto assignment to Go wire type. +func AssignmentFromProto(p *master_pb.BlockVolumeAssignment) BlockVolumeAssignment { + if p == nil { + return BlockVolumeAssignment{} + } + return BlockVolumeAssignment{ + Path: p.Path, + Epoch: p.Epoch, + Role: p.Role, + LeaseTtlMs: p.LeaseTtlMs, + } +} + +// AssignmentsFromProto converts a slice of proto assignments to Go wire types. +func AssignmentsFromProto(protos []*master_pb.BlockVolumeAssignment) []BlockVolumeAssignment { + out := make([]BlockVolumeAssignment, len(protos)) + for i, p := range protos { + out[i] = AssignmentFromProto(p) + } + return out +} diff --git a/weed/storage/blockvol/block_heartbeat_proto_test.go b/weed/storage/blockvol/block_heartbeat_proto_test.go new file mode 100644 index 000000000..3c1b55da2 --- /dev/null +++ b/weed/storage/blockvol/block_heartbeat_proto_test.go @@ -0,0 +1,85 @@ +package blockvol + +import ( + "testing" +) + +func TestInfoMessageRoundTrip(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/vol1.blk", + VolumeSize: 1 << 30, + BlockSize: 4096, + Epoch: 42, + Role: RoleToWire(RolePrimary), + WalHeadLsn: 1000, + CheckpointLsn: 900, + HasLease: true, + DiskType: "ssd", + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if back != orig { + t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig) + } +} + +func TestShortInfoRoundTrip(t *testing.T) { + orig := BlockVolumeShortInfoMessage{ + Path: "/data/vol2.blk", + VolumeSize: 2 << 30, + BlockSize: 4096, + DiskType: "hdd", + } + pb := ShortInfoToProto(orig) + back := ShortInfoFromProto(pb) + if back != orig { + t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig) + } +} + +func TestAssignmentRoundTrip(t *testing.T) { + orig := BlockVolumeAssignment{ + Path: "/data/vol3.blk", + Epoch: 7, + Role: RoleToWire(RoleReplica), + LeaseTtlMs: 5000, + } + pb := AssignmentToProto(orig) + back := AssignmentFromProto(pb) + if back != orig { + t.Fatalf("round-trip mismatch:\n got %+v\n want %+v", back, orig) + } +} + +func TestInfoMessagesSliceRoundTrip(t *testing.T) { + origSlice := []BlockVolumeInfoMessage{ + {Path: "/a.blk", VolumeSize: 100, Epoch: 1}, + {Path: "/b.blk", VolumeSize: 200, Epoch: 2, HasLease: true}, + } + pbs := InfoMessagesToProto(origSlice) + back := InfoMessagesFromProto(pbs) + if len(back) != len(origSlice) { + t.Fatalf("length mismatch: got %d, want %d", len(back), len(origSlice)) + } + for i := range origSlice { + if back[i] != origSlice[i] { + t.Fatalf("index %d mismatch:\n got %+v\n want %+v", i, back[i], origSlice[i]) + } + } +} + +func TestNilProtoConversions(t *testing.T) { + // Nil proto -> zero-value Go types. + info := InfoMessageFromProto(nil) + if info != (BlockVolumeInfoMessage{}) { + t.Fatalf("nil info proto should yield zero value, got %+v", info) + } + short := ShortInfoFromProto(nil) + if short != (BlockVolumeShortInfoMessage{}) { + t.Fatalf("nil short proto should yield zero value, got %+v", short) + } + assign := AssignmentFromProto(nil) + if assign != (BlockVolumeAssignment{}) { + t.Fatalf("nil assignment proto should yield zero value, got %+v", assign) + } +} diff --git a/weed/storage/blockvol/csi/cmd/block-csi/main.go b/weed/storage/blockvol/csi/cmd/block-csi/main.go new file mode 100644 index 000000000..41eb56864 --- /dev/null +++ b/weed/storage/blockvol/csi/cmd/block-csi/main.go @@ -0,0 +1,68 @@ +// block-csi is the SeaweedFS BlockVol CSI driver. +// It embeds a BlockVol engine and iSCSI target in-process, serving +// CSI Identity, Controller, and Node services on a Unix socket. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + blockcsi "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/csi" +) + +func main() { + endpoint := flag.String("endpoint", "unix:///csi/csi.sock", "CSI endpoint (unix socket)") + dataDir := flag.String("data-dir", "/var/lib/sw-block", "volume data directory") + iscsiAddr := flag.String("iscsi-addr", "127.0.0.1:3260", "local iSCSI target listen address") + iqnPrefix := flag.String("iqn-prefix", "iqn.2024.com.seaweedfs", "IQN prefix for volumes") + nodeID := flag.String("node-id", "", "node identifier (required)") + masterAddr := flag.String("master", "", "master address for control-plane mode (e.g. master:9333)") + mode := flag.String("mode", "all", "driver mode: controller, node, or all") + flag.Parse() + + if *nodeID == "" { + fmt.Fprintln(os.Stderr, "error: -node-id is required") + flag.Usage() + os.Exit(1) + } + if *mode == "controller" && *masterAddr == "" { + fmt.Fprintln(os.Stderr, "error: -master is required in controller mode") + flag.Usage() + os.Exit(1) + } + + logger := log.New(os.Stdout, "[block-csi] ", log.LstdFlags) + + driver, err := blockcsi.NewCSIDriver(blockcsi.DriverConfig{ + Endpoint: *endpoint, + DataDir: *dataDir, + ISCSIAddr: *iscsiAddr, + IQNPrefix: *iqnPrefix, + NodeID: *nodeID, + MasterAddr: *masterAddr, + Mode: *mode, + Logger: logger, + }) + if err != nil { + log.Fatalf("create CSI driver: %v", err) + } + + // Graceful shutdown on signal. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigCh + logger.Printf("received %v, shutting down...", sig) + driver.Stop() + }() + + logger.Printf("starting block-csi driver: node=%s endpoint=%s", *nodeID, *endpoint) + if err := driver.Run(); err != nil { + log.Fatalf("CSI driver: %v", err) + } + logger.Println("block-csi driver stopped") +} diff --git a/weed/storage/blockvol/csi/cmd/block-csi/smoke-test.sh b/weed/storage/blockvol/csi/cmd/block-csi/smoke-test.sh new file mode 100644 index 000000000..9b536b7de --- /dev/null +++ b/weed/storage/blockvol/csi/cmd/block-csi/smoke-test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# smoke-test.sh -- deploy sw-block CSI driver to k3s/kind and verify PVC lifecycle. +# Requires: k3s or kind pre-installed, kubectl, go, docker (for kind). +# Usage: bash smoke-test.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../../.." && pwd)" +DEPLOY_DIR="$SCRIPT_DIR/../../deploy" +BINARY="$SCRIPT_DIR/block-csi" + +echo "=== sw-block CSI smoke test ===" + +# 1. Build binary +echo "[1/7] Building block-csi (linux/amd64)..." +cd "$REPO_ROOT" +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o "$BINARY" ./weed/storage/blockvol/csi/cmd/block-csi/ +echo " built: $BINARY" + +# 2. Detect runtime (k3s or kind) +if command -v k3s &>/dev/null; then + RUNTIME=k3s + KUBECTL="k3s kubectl" + echo "[2/7] Detected k3s" + + # Copy binary to a path accessible by the k3s node + sudo cp "$BINARY" /usr/local/bin/block-csi + sudo chmod +x /usr/local/bin/block-csi +elif command -v kind &>/dev/null; then + RUNTIME=kind + KUBECTL="kubectl" + echo "[2/7] Detected kind" + + # Build a minimal container image + cat > /tmp/block-csi-Dockerfile <<'DOCKERFILE' +FROM alpine:3.19 +RUN apk add --no-cache open-iscsi e2fsprogs util-linux +COPY block-csi /usr/local/bin/block-csi +ENTRYPOINT ["/usr/local/bin/block-csi"] +DOCKERFILE + docker build -t sw-block-csi:local -f /tmp/block-csi-Dockerfile "$SCRIPT_DIR" + kind load docker-image sw-block-csi:local +else + echo "ERROR: neither k3s nor kind found" >&2 + exit 1 +fi + +# 3. Deploy manifests +echo "[3/7] Deploying CSI driver..." +$KUBECTL apply -f "$DEPLOY_DIR/rbac.yaml" +$KUBECTL apply -f "$DEPLOY_DIR/csi-driver.yaml" +$KUBECTL apply -f "$DEPLOY_DIR/storageclass.yaml" + +echo " Waiting for DaemonSet to be ready..." +$KUBECTL -n kube-system rollout status daemonset/sw-block-csi-node --timeout=120s + +# 4. Create PVC +echo "[4/7] Creating PVC..." +$KUBECTL apply -f "$DEPLOY_DIR/example-pvc.yaml" + +echo " Waiting for pod to be ready..." +$KUBECTL wait --for=condition=Ready pod/sw-block-test-pod --timeout=120s + +# 5. Verify data +echo "[5/7] Verifying data..." +DATA=$($KUBECTL exec sw-block-test-pod -- cat /data/test.txt) +if [ "$DATA" = "hello sw-block" ]; then + echo " OK: data verified" +else + echo " FAIL: expected 'hello sw-block', got '$DATA'" >&2 + exit 1 +fi + +# 6. Delete and recreate pod to verify persistence +echo "[6/7] Deleting pod and verifying persistence..." +$KUBECTL delete pod sw-block-test-pod --grace-period=5 +$KUBECTL apply -f - <<'EOF' +apiVersion: v1 +kind: Pod +metadata: + name: sw-block-test-pod +spec: + containers: + - name: app + image: busybox + command: ["sh", "-c", "cat /data/test.txt && sleep 3600"] + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: sw-block-test +EOF + +$KUBECTL wait --for=condition=Ready pod/sw-block-test-pod --timeout=120s +DATA=$($KUBECTL exec sw-block-test-pod -- cat /data/test.txt) +if [ "$DATA" = "hello sw-block" ]; then + echo " OK: data persisted across pod restart" +else + echo " FAIL: data not persisted, got '$DATA'" >&2 + exit 1 +fi + +# 7. Cleanup +echo "[7/7] Cleaning up..." +$KUBECTL delete pod sw-block-test-pod --grace-period=5 || true +$KUBECTL delete pvc sw-block-test || true +$KUBECTL delete -f "$DEPLOY_DIR/csi-driver.yaml" || true +$KUBECTL delete -f "$DEPLOY_DIR/storageclass.yaml" || true +$KUBECTL delete -f "$DEPLOY_DIR/rbac.yaml" || true + +echo "=== smoke test PASSED ===" diff --git a/weed/storage/blockvol/csi/controller.go b/weed/storage/blockvol/csi/controller.go new file mode 100644 index 000000000..d37da47ae --- /dev/null +++ b/weed/storage/blockvol/csi/controller.go @@ -0,0 +1,130 @@ +package csi + +import ( + "context" + "errors" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + defaultVolumeSizeBytes = 1 << 30 // 1 GiB + minVolumeSizeBytes = 1 << 20 // 1 MiB + blockSize = 4096 +) + +type controllerServer struct { + csi.UnimplementedControllerServer + backend VolumeBackend +} + +func (s *controllerServer) CreateVolume(_ context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + if req.Name == "" { + return nil, status.Error(codes.InvalidArgument, "volume name is required") + } + if len(req.VolumeCapabilities) == 0 { + return nil, status.Error(codes.InvalidArgument, "volume capabilities are required") + } + + sizeBytes := int64(defaultVolumeSizeBytes) + if req.CapacityRange != nil && req.CapacityRange.RequiredBytes > 0 { + sizeBytes = req.CapacityRange.RequiredBytes + } else if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 { + // No RequiredBytes set — use LimitBytes as the target size. + sizeBytes = req.CapacityRange.LimitBytes + } + if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 { + if req.CapacityRange.RequiredBytes > req.CapacityRange.LimitBytes { + return nil, status.Errorf(codes.InvalidArgument, + "required_bytes (%d) exceeds limit_bytes (%d)", + req.CapacityRange.RequiredBytes, req.CapacityRange.LimitBytes) + } + } + if sizeBytes < minVolumeSizeBytes { + sizeBytes = minVolumeSizeBytes + } + // Round up to block size. + if sizeBytes%blockSize != 0 { + sizeBytes = (sizeBytes/blockSize + 1) * blockSize + } + // Verify rounded size still respects LimitBytes. + if req.CapacityRange != nil && req.CapacityRange.LimitBytes > 0 { + if sizeBytes > req.CapacityRange.LimitBytes { + return nil, status.Errorf(codes.InvalidArgument, + "volume size (%d) after rounding exceeds limit_bytes (%d)", + sizeBytes, req.CapacityRange.LimitBytes) + } + } + + info, err := s.backend.CreateVolume(context.Background(), req.Name, uint64(sizeBytes)) + if err != nil { + if errors.Is(err, ErrVolumeSizeMismatch) { + return nil, status.Errorf(codes.AlreadyExists, "volume %q exists with different size", req.Name) + } + return nil, status.Errorf(codes.Internal, "create volume: %v", err) + } + + resp := &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: info.VolumeID, + CapacityBytes: int64(info.CapacityBytes), + }, + } + + // Attach volume_context with iSCSI target info for NodeStageVolume. + if info.ISCSIAddr != "" || info.IQN != "" { + resp.Volume.VolumeContext = map[string]string{ + "iscsiAddr": info.ISCSIAddr, + "iqn": info.IQN, + } + } + + return resp, nil +} + +func (s *controllerServer) DeleteVolume(_ context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + + // Idempotent: DeleteVolume succeeds even if volume doesn't exist. + if err := s.backend.DeleteVolume(context.Background(), req.VolumeId); err != nil { + return nil, status.Errorf(codes.Internal, "delete volume: %v", err) + } + + return &csi.DeleteVolumeResponse{}, nil +} + +func (s *controllerServer) ControllerGetCapabilities(_ context.Context, _ *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + return &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + }, + }, nil +} + +func (s *controllerServer) ValidateVolumeCapabilities(_ context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if len(req.VolumeCapabilities) == 0 { + return nil, status.Error(codes.InvalidArgument, "volume capabilities are required") + } + if _, err := s.backend.LookupVolume(context.Background(), req.VolumeId); err != nil { + return nil, status.Errorf(codes.NotFound, "volume %q not found", req.VolumeId) + } + + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ + VolumeCapabilities: req.VolumeCapabilities, + }, + }, nil +} diff --git a/weed/storage/blockvol/csi/controller_test.go b/weed/storage/blockvol/csi/controller_test.go new file mode 100644 index 000000000..89e2df4aa --- /dev/null +++ b/weed/storage/blockvol/csi/controller_test.go @@ -0,0 +1,127 @@ +package csi + +import ( + "context" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// testVolCaps returns a standard volume capability for testing. +func testVolCaps() []*csi.VolumeCapability { + return []*csi.VolumeCapability{{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }} +} + +func testVolCap() *csi.VolumeCapability { + return testVolCaps()[0] +} + +func TestController_CreateVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "test-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 4 * 1024 * 1024, + }, + }) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + if resp.Volume.VolumeId != "test-vol" { + t.Fatalf("volume_id: got %q, want %q", resp.Volume.VolumeId, "test-vol") + } + if resp.Volume.CapacityBytes != 4*1024*1024 { + t.Fatalf("capacity: got %d, want %d", resp.Volume.CapacityBytes, 4*1024*1024) + } + if !mgr.VolumeExists("test-vol") { + t.Fatal("expected volume to exist") + } + + // Verify volume_context has iSCSI info. + if resp.Volume.VolumeContext == nil { + t.Fatal("expected volume_context to be set") + } + if resp.Volume.VolumeContext["iqn"] == "" { + t.Fatal("expected iqn in volume_context") + } +} + +func TestController_CreateIdempotent(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + req := &csi.CreateVolumeRequest{ + Name: "idem-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 4 * 1024 * 1024, + }, + } + + if _, err := cs.CreateVolume(context.Background(), req); err != nil { + t.Fatalf("first create: %v", err) + } + + // Second create with same size should succeed (idempotent). + resp, err := cs.CreateVolume(context.Background(), req) + if err != nil { + t.Fatalf("second create: %v", err) + } + if resp.Volume.VolumeId != "idem-vol" { + t.Fatalf("volume_id: got %q, want %q", resp.Volume.VolumeId, "idem-vol") + } +} + +func TestController_DeleteVolume(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Create then delete. + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "del-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 4 * 1024 * 1024, + }, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + _, err = cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{ + VolumeId: "del-vol", + }) + if err != nil { + t.Fatalf("delete: %v", err) + } + if mgr.VolumeExists("del-vol") { + t.Fatal("expected volume to not exist after delete") + } +} + +func TestController_DeleteNotFound(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + cs := &controllerServer{backend: backend} + + // Delete non-existent volume -- should succeed (CSI spec idempotency). + _, err := cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{ + VolumeId: "nonexistent", + }) + if err != nil { + t.Fatalf("delete non-existent: %v", err) + } +} diff --git a/weed/storage/blockvol/csi/deploy/csi-controller.yaml b/weed/storage/blockvol/csi/deploy/csi-controller.yaml new file mode 100644 index 000000000..2639077e4 --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/csi-controller.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sw-block-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: sw-block-csi-controller + template: + metadata: + labels: + app: sw-block-csi-controller + spec: + serviceAccountName: sw-block-csi + containers: + - name: block-csi + image: sw-block-csi:local + imagePullPolicy: Never + args: + - "-endpoint=unix:///csi/csi.sock" + - "-mode=controller" + - "-master=master:9333" + - "-node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 + args: + - "--csi-address=/csi/csi.sock" + - "--feature-gates=Topology=true" + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/weed/storage/blockvol/csi/deploy/csi-driver.yaml b/weed/storage/blockvol/csi/deploy/csi-driver.yaml new file mode 100644 index 000000000..1e8b48d2a --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/csi-driver.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: block.csi.seaweedfs.com +spec: + attachRequired: false + podInfoOnMount: false + volumeLifecycleModes: + - Persistent diff --git a/weed/storage/blockvol/csi/deploy/csi-node.yaml b/weed/storage/blockvol/csi/deploy/csi-node.yaml new file mode 100644 index 000000000..9a222719c --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/csi-node.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sw-block-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: sw-block-csi-node + template: + metadata: + labels: + app: sw-block-csi-node + spec: + serviceAccountName: sw-block-csi + hostNetwork: true + hostPID: true + containers: + - name: block-csi + image: sw-block-csi:local + imagePullPolicy: Never + args: + - "-endpoint=unix:///csi/csi.sock" + - "-mode=node" + - "-node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + - name: dev + mountPath: /dev + - name: iscsi-dir + mountPath: /etc/iscsi + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.12.0 + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/block.csi.seaweedfs.com/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/block.csi.seaweedfs.com + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: iscsi-dir + hostPath: + path: /etc/iscsi + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory diff --git a/weed/storage/blockvol/csi/deploy/example-pvc.yaml b/weed/storage/blockvol/csi/deploy/example-pvc.yaml new file mode 100644 index 000000000..571fc5139 --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/example-pvc.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sw-block-test +spec: + accessModes: + - ReadWriteOnce + storageClassName: sw-block + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: sw-block-test-pod +spec: + containers: + - name: app + image: busybox + command: ["sh", "-c", "echo 'hello sw-block' > /data/test.txt && cat /data/test.txt && sleep 3600"] + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: sw-block-test diff --git a/weed/storage/blockvol/csi/deploy/rbac.yaml b/weed/storage/blockvol/csi/deploy/rbac.yaml new file mode 100644 index 000000000..9807dfc6e --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/rbac.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sw-block-csi + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sw-block-csi +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sw-block-csi +subjects: + - kind: ServiceAccount + name: sw-block-csi + namespace: kube-system +roleRef: + kind: ClusterRole + name: sw-block-csi + apiGroup: rbac.authorization.k8s.io diff --git a/weed/storage/blockvol/csi/deploy/storageclass.yaml b/weed/storage/blockvol/csi/deploy/storageclass.yaml new file mode 100644 index 000000000..2bdd3432c --- /dev/null +++ b/weed/storage/blockvol/csi/deploy/storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: sw-block +provisioner: block.csi.seaweedfs.com +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete diff --git a/weed/storage/blockvol/csi/identity.go b/weed/storage/blockvol/csi/identity.go new file mode 100644 index 000000000..eef911111 --- /dev/null +++ b/weed/storage/blockvol/csi/identity.go @@ -0,0 +1,44 @@ +package csi + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const ( + DriverName = "block.csi.seaweedfs.com" + DriverVersion = "0.1.0" +) + +type identityServer struct { + csi.UnimplementedIdentityServer +} + +func (s *identityServer) GetPluginInfo(_ context.Context, _ *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + return &csi.GetPluginInfoResponse{ + Name: DriverName, + VendorVersion: DriverVersion, + }, nil +} + +func (s *identityServer) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + }, + }, nil +} + +func (s *identityServer) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{ + Ready: wrapperspb.Bool(true), + }, nil +} diff --git a/weed/storage/blockvol/csi/identity_test.go b/weed/storage/blockvol/csi/identity_test.go new file mode 100644 index 000000000..f242397b6 --- /dev/null +++ b/weed/storage/blockvol/csi/identity_test.go @@ -0,0 +1,51 @@ +package csi + +import ( + "context" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +func TestIdentity_GetPluginInfo(t *testing.T) { + s := &identityServer{} + resp, err := s.GetPluginInfo(context.Background(), &csi.GetPluginInfoRequest{}) + if err != nil { + t.Fatalf("GetPluginInfo: %v", err) + } + if resp.Name != DriverName { + t.Fatalf("name: got %q, want %q", resp.Name, DriverName) + } + if resp.VendorVersion != DriverVersion { + t.Fatalf("version: got %q, want %q", resp.VendorVersion, DriverVersion) + } +} + +func TestIdentity_GetPluginCapabilities(t *testing.T) { + s := &identityServer{} + resp, err := s.GetPluginCapabilities(context.Background(), &csi.GetPluginCapabilitiesRequest{}) + if err != nil { + t.Fatalf("GetPluginCapabilities: %v", err) + } + if len(resp.Capabilities) != 1 { + t.Fatalf("capabilities: got %d, want 1", len(resp.Capabilities)) + } + svc := resp.Capabilities[0].GetService() + if svc == nil { + t.Fatal("expected service capability") + } + if svc.Type != csi.PluginCapability_Service_CONTROLLER_SERVICE { + t.Fatalf("capability type: got %v, want CONTROLLER_SERVICE", svc.Type) + } +} + +func TestIdentity_Probe(t *testing.T) { + s := &identityServer{} + resp, err := s.Probe(context.Background(), &csi.ProbeRequest{}) + if err != nil { + t.Fatalf("Probe: %v", err) + } + if resp.Ready == nil || !resp.Ready.Value { + t.Fatal("expected ready=true") + } +} diff --git a/weed/storage/blockvol/csi/iscsi_util.go b/weed/storage/blockvol/csi/iscsi_util.go new file mode 100644 index 000000000..a7d9f8b94 --- /dev/null +++ b/weed/storage/blockvol/csi/iscsi_util.go @@ -0,0 +1,306 @@ +package csi + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// ISCSIUtil provides iSCSI initiator operations. +type ISCSIUtil interface { + Discovery(ctx context.Context, portal string) error + Login(ctx context.Context, iqn, portal string) error + Logout(ctx context.Context, iqn string) error + GetDeviceByIQN(ctx context.Context, iqn string) (string, error) + IsLoggedIn(ctx context.Context, iqn string) (bool, error) +} + +// MountUtil provides filesystem mount operations. +type MountUtil interface { + FormatAndMount(ctx context.Context, device, target, fsType string) error + Mount(ctx context.Context, source, target, fsType string, readOnly bool) error + BindMount(ctx context.Context, source, target string, readOnly bool) error + Unmount(ctx context.Context, target string) error + IsFormatted(ctx context.Context, device string) (bool, error) + IsMounted(ctx context.Context, target string) (bool, error) +} + +// realISCSIUtil uses iscsiadm CLI. +type realISCSIUtil struct{} + +func (r *realISCSIUtil) Discovery(ctx context.Context, portal string) error { + cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "discovery", "-t", "sendtargets", "-p", portal) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("iscsiadm discovery: %s: %w", string(out), err) + } + return nil +} + +func (r *realISCSIUtil) Login(ctx context.Context, iqn, portal string) error { + cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "node", "-T", iqn, "-p", portal, "--login") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("iscsiadm login: %s: %w", string(out), err) + } + return nil +} + +func (r *realISCSIUtil) Logout(ctx context.Context, iqn string) error { + cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "node", "-T", iqn, "--logout") + out, err := cmd.CombinedOutput() + if err != nil { + // Treat "not logged in" as success. + if strings.Contains(string(out), "No matching sessions") { + return nil + } + return fmt.Errorf("iscsiadm logout: %s: %w", string(out), err) + } + return nil +} + +func (r *realISCSIUtil) GetDeviceByIQN(ctx context.Context, iqn string) (string, error) { + // Poll for device to appear (iSCSI login is async). + deadline := time.After(10 * time.Second) + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-deadline: + return "", fmt.Errorf("timeout waiting for device for IQN %s", iqn) + case <-ticker.C: + // Look for block device symlinks under /dev/disk/by-path/ + pattern := fmt.Sprintf("/dev/disk/by-path/*%s*", iqn) + matches, err := filepath.Glob(pattern) + if err != nil { + continue + } + for _, m := range matches { + // Skip partitions. + if strings.Contains(m, "-part") { + continue + } + // Resolve symlink to actual device. + dev, err := filepath.EvalSymlinks(m) + if err != nil { + continue + } + return dev, nil + } + } + } +} + +func (r *realISCSIUtil) IsLoggedIn(ctx context.Context, iqn string) (bool, error) { + cmd := exec.CommandContext(ctx, "iscsiadm", "-m", "session") + out, err := cmd.CombinedOutput() + if err != nil { + // Exit code 21 = no sessions, not an error. + outStr := string(out) + if strings.Contains(outStr, "No active sessions") { + return false, nil + } + // Also handle exit code 21 directly (nsenter may suppress output). + if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 21 { + return false, nil + } + return false, fmt.Errorf("iscsiadm session: %s: %w", outStr, err) + } + return strings.Contains(string(out), iqn), nil +} + +// realMountUtil uses mount/umount/mkfs CLI. +type realMountUtil struct{} + +func (r *realMountUtil) FormatAndMount(ctx context.Context, device, target, fsType string) error { + formatted, err := r.IsFormatted(ctx, device) + if err != nil { + return err + } + if !formatted { + cmd := exec.CommandContext(ctx, "mkfs."+fsType, device) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("mkfs.%s: %s: %w", fsType, string(out), err) + } + } + return r.Mount(ctx, device, target, fsType, false) +} + +func (r *realMountUtil) Mount(ctx context.Context, source, target, fsType string, readOnly bool) error { + args := []string{"-t", fsType} + if readOnly { + args = append(args, "-o", "ro") + } + args = append(args, source, target) + cmd := exec.CommandContext(ctx, "mount", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("mount: %s: %w", string(out), err) + } + return nil +} + +func (r *realMountUtil) BindMount(ctx context.Context, source, target string, readOnly bool) error { + args := []string{"--bind", source, target} + cmd := exec.CommandContext(ctx, "mount", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("bind mount: %s: %w", string(out), err) + } + if readOnly { + cmd = exec.CommandContext(ctx, "mount", "-o", "remount,bind,ro", target) + out, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("remount ro: %s: %w", string(out), err) + } + } + return nil +} + +func (r *realMountUtil) Unmount(ctx context.Context, target string) error { + cmd := exec.CommandContext(ctx, "umount", target) + out, err := cmd.CombinedOutput() + if err != nil { + // Treat "not mounted" as success. + if strings.Contains(string(out), "not mounted") { + return nil + } + return fmt.Errorf("umount: %s: %w", string(out), err) + } + return nil +} + +func (r *realMountUtil) IsFormatted(ctx context.Context, device string) (bool, error) { + cmd := exec.CommandContext(ctx, "blkid", "-p", device) + out, err := cmd.CombinedOutput() + if err != nil { + // Exit code 2 = no filesystem found. + if cmd.ProcessState != nil && cmd.ProcessState.ExitCode() == 2 { + return false, nil + } + return false, fmt.Errorf("blkid: %s: %w", string(out), err) + } + return strings.Contains(string(out), "TYPE="), nil +} + +func (r *realMountUtil) IsMounted(ctx context.Context, target string) (bool, error) { + cmd := exec.CommandContext(ctx, "mountpoint", "-q", target) + err := cmd.Run() + if err == nil { + return true, nil + } + // Non-zero exit = not a mount point. + return false, nil +} + +// mockISCSIUtil is a test double for ISCSIUtil. +type mockISCSIUtil struct { + discoveryErr error + loginErr error + logoutErr error + getDeviceResult string + getDeviceErr error + loggedIn map[string]bool + calls []string +} + +func newMockISCSIUtil() *mockISCSIUtil { + return &mockISCSIUtil{loggedIn: make(map[string]bool)} +} + +func (m *mockISCSIUtil) Discovery(_ context.Context, portal string) error { + m.calls = append(m.calls, "discovery:"+portal) + return m.discoveryErr +} + +func (m *mockISCSIUtil) Login(_ context.Context, iqn, portal string) error { + m.calls = append(m.calls, "login:"+iqn) + if m.loginErr != nil { + return m.loginErr + } + m.loggedIn[iqn] = true + return nil +} + +func (m *mockISCSIUtil) Logout(_ context.Context, iqn string) error { + m.calls = append(m.calls, "logout:"+iqn) + if m.logoutErr != nil { + return m.logoutErr + } + delete(m.loggedIn, iqn) + return nil +} + +func (m *mockISCSIUtil) GetDeviceByIQN(_ context.Context, iqn string) (string, error) { + m.calls = append(m.calls, "getdevice:"+iqn) + return m.getDeviceResult, m.getDeviceErr +} + +func (m *mockISCSIUtil) IsLoggedIn(_ context.Context, iqn string) (bool, error) { + return m.loggedIn[iqn], nil +} + +// mockMountUtil is a test double for MountUtil. +type mockMountUtil struct { + formatAndMountErr error + mountErr error + unmountErr error + isFormattedResult bool + isMountedTargets map[string]bool + calls []string +} + +func newMockMountUtil() *mockMountUtil { + return &mockMountUtil{isMountedTargets: make(map[string]bool)} +} + +func (m *mockMountUtil) FormatAndMount(_ context.Context, device, target, fsType string) error { + m.calls = append(m.calls, "formatandmount:"+device+":"+target) + if m.formatAndMountErr != nil { + return m.formatAndMountErr + } + m.isMountedTargets[target] = true + return nil +} + +func (m *mockMountUtil) Mount(_ context.Context, source, target, fsType string, readOnly bool) error { + m.calls = append(m.calls, "mount:"+source+":"+target) + if m.mountErr != nil { + return m.mountErr + } + m.isMountedTargets[target] = true + return nil +} + +func (m *mockMountUtil) BindMount(_ context.Context, source, target string, readOnly bool) error { + m.calls = append(m.calls, "bindmount:"+source+":"+target) + if m.mountErr != nil { + return m.mountErr + } + m.isMountedTargets[target] = true + return nil +} + +func (m *mockMountUtil) Unmount(_ context.Context, target string) error { + m.calls = append(m.calls, "unmount:"+target) + if m.unmountErr != nil { + return m.unmountErr + } + delete(m.isMountedTargets, target) + return nil +} + +func (m *mockMountUtil) IsFormatted(_ context.Context, device string) (bool, error) { + return m.isFormattedResult, nil +} + +func (m *mockMountUtil) IsMounted(_ context.Context, target string) (bool, error) { + return m.isMountedTargets[target], nil +} diff --git a/weed/storage/blockvol/csi/node.go b/weed/storage/blockvol/csi/node.go new file mode 100644 index 000000000..43182b415 --- /dev/null +++ b/weed/storage/blockvol/csi/node.go @@ -0,0 +1,313 @@ +package csi + +import ( + "context" + "fmt" + "log" + "os" + "sync" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// stagedVolumeInfo tracks info needed for NodeUnstageVolume. +type stagedVolumeInfo struct { + iqn string + iscsiAddr string + isLocal bool // true if volume is served by local VolumeManager +} + +type nodeServer struct { + csi.UnimplementedNodeServer + mgr *VolumeManager // may be nil in controller-only mode + nodeID string + iqnPrefix string // for IQN derivation fallback on restart + iscsiUtil ISCSIUtil + mountUtil MountUtil + logger *log.Logger + + stagedMu sync.Mutex + staged map[string]*stagedVolumeInfo // volumeID -> staged info +} + +func (s *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + volumeID := req.VolumeId + stagingPath := req.StagingTargetPath + + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if stagingPath == "" { + return nil, status.Error(codes.InvalidArgument, "staging target path is required") + } + if req.VolumeCapability == nil { + return nil, status.Error(codes.InvalidArgument, "volume capability is required") + } + + // Idempotency: if already mounted at staging path, return OK. + mounted, err := s.mountUtil.IsMounted(ctx, stagingPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "check mount: %v", err) + } + if mounted { + s.logger.Printf("NodeStageVolume: %s already mounted at %s", volumeID, stagingPath) + return &csi.NodeStageVolumeResponse{}, nil + } + + // Determine iSCSI target info: from volume_context (remote) or local mgr. + var iqn, portal string + isLocal := false + + if req.VolumeContext != nil && req.VolumeContext["iscsiAddr"] != "" && req.VolumeContext["iqn"] != "" { + // Remote target: iSCSI info from volume_context (set by controller via master). + portal = req.VolumeContext["iscsiAddr"] + iqn = req.VolumeContext["iqn"] + } else if s.mgr != nil { + // Local fallback: open volume via local VolumeManager. + isLocal = true + if err := s.mgr.OpenVolume(volumeID); err != nil { + return nil, status.Errorf(codes.Internal, "open volume: %v", err) + } + iqn = s.mgr.VolumeIQN(volumeID) + portal = s.mgr.ListenAddr() + } else { + return nil, status.Error(codes.FailedPrecondition, "no volume_context and no local volume manager") + } + + // Cleanup on error. + success := false + defer func() { + if !success { + s.logger.Printf("NodeStageVolume: cleaning up %s after error", volumeID) + if isLocal && s.mgr != nil { + s.mgr.CloseVolume(volumeID) + } + } + }() + + // Check if already logged in, skip login if so. + loggedIn, err := s.iscsiUtil.IsLoggedIn(ctx, iqn) + if err != nil { + return nil, status.Errorf(codes.Internal, "check iscsi login: %v", err) + } + + if !loggedIn { + // Discovery + login. + if err := s.iscsiUtil.Discovery(ctx, portal); err != nil { + return nil, status.Errorf(codes.Internal, "iscsi discovery: %v", err) + } + if err := s.iscsiUtil.Login(ctx, iqn, portal); err != nil { + return nil, status.Errorf(codes.Internal, "iscsi login: %v", err) + } + } + + // Wait for device to appear. + device, err := s.iscsiUtil.GetDeviceByIQN(ctx, iqn) + if err != nil { + return nil, status.Errorf(codes.Internal, "get device: %v", err) + } + + // Ensure staging directory exists. + if err := os.MkdirAll(stagingPath, 0750); err != nil { + return nil, status.Errorf(codes.Internal, "create staging dir: %v", err) + } + + // Format (if needed) and mount. + fsType := "ext4" + if req.VolumeCapability != nil { + if mnt := req.VolumeCapability.GetMount(); mnt != nil && mnt.FsType != "" { + fsType = mnt.FsType + } + } + + if err := s.mountUtil.FormatAndMount(ctx, device, stagingPath, fsType); err != nil { + return nil, status.Errorf(codes.Internal, "format and mount: %v", err) + } + + // Track staged volume for unstage. + s.stagedMu.Lock() + if s.staged == nil { + s.staged = make(map[string]*stagedVolumeInfo) + } + s.staged[volumeID] = &stagedVolumeInfo{ + iqn: iqn, + iscsiAddr: portal, + isLocal: isLocal, + } + s.stagedMu.Unlock() + + success = true + s.logger.Printf("NodeStageVolume: %s staged at %s (device=%s, iqn=%s, local=%v)", volumeID, stagingPath, device, iqn, isLocal) + return &csi.NodeStageVolumeResponse{}, nil +} + +func (s *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + volumeID := req.VolumeId + stagingPath := req.StagingTargetPath + + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if stagingPath == "" { + return nil, status.Error(codes.InvalidArgument, "staging target path is required") + } + + // Look up staged info. If not found (e.g. driver restarted), derive IQN. + s.stagedMu.Lock() + info := s.staged[volumeID] + s.stagedMu.Unlock() + + var iqn string + isLocal := false + if info != nil { + iqn = info.iqn + isLocal = info.isLocal + } else { + // Restart fallback: derive IQN from volumeID. + // iscsiadm -m node -T --logout works without knowing the portal. + if s.mgr != nil { + iqn = s.mgr.VolumeIQN(volumeID) + isLocal = true + } else if s.iqnPrefix != "" { + iqn = s.iqnPrefix + ":" + blockvol.SanitizeIQN(volumeID) + } + s.logger.Printf("NodeUnstageVolume: %s not in staged map, derived iqn=%s", volumeID, iqn) + } + + // Best-effort cleanup: always attempt all steps even if one fails. + var firstErr error + + // Unmount. + if err := s.mountUtil.Unmount(ctx, stagingPath); err != nil { + s.logger.Printf("NodeUnstageVolume: unmount error: %v", err) + firstErr = err + } + + // iSCSI logout. + if iqn != "" { + if err := s.iscsiUtil.Logout(ctx, iqn); err != nil { + s.logger.Printf("NodeUnstageVolume: logout error: %v", err) + if firstErr == nil { + firstErr = err + } + } + } + + // Close the local volume if applicable. + if isLocal && s.mgr != nil { + if err := s.mgr.CloseVolume(volumeID); err != nil { + s.logger.Printf("NodeUnstageVolume: close volume error: %v", err) + if firstErr == nil { + firstErr = err + } + } + } + + if firstErr != nil { + // Keep staged entry so retry has correct isLocal/iqn info. + return nil, status.Errorf(codes.Internal, "unstage: %v", firstErr) + } + + // Remove from staged map only after successful cleanup. + s.stagedMu.Lock() + delete(s.staged, volumeID) + s.stagedMu.Unlock() + + s.logger.Printf("NodeUnstageVolume: %s unstaged from %s", volumeID, stagingPath) + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (s *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + volumeID := req.VolumeId + targetPath := req.TargetPath + stagingPath := req.StagingTargetPath + + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if targetPath == "" { + return nil, status.Error(codes.InvalidArgument, "target path is required") + } + if stagingPath == "" { + return nil, status.Error(codes.InvalidArgument, "staging target path is required") + } + + // Idempotency: if already bind-mounted, return OK. + mounted, err := s.mountUtil.IsMounted(ctx, targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "check mount: %v", err) + } + if mounted { + s.logger.Printf("NodePublishVolume: %s already mounted at %s", volumeID, targetPath) + return &csi.NodePublishVolumeResponse{}, nil + } + + // Ensure target directory exists. + if err := os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Errorf(codes.Internal, "create target dir: %v", err) + } + + // Bind mount staging path to target path. + readOnly := req.Readonly + if err := s.mountUtil.BindMount(ctx, stagingPath, targetPath, readOnly); err != nil { + return nil, status.Errorf(codes.Internal, "bind mount: %v", err) + } + + s.logger.Printf("NodePublishVolume: %s published at %s", volumeID, targetPath) + return &csi.NodePublishVolumeResponse{}, nil +} + +func (s *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + if req.VolumeId == "" { + return nil, status.Error(codes.InvalidArgument, "volume ID is required") + } + if req.TargetPath == "" { + return nil, status.Error(codes.InvalidArgument, "target path is required") + } + + // Idempotent: only unmount if still mounted. + mounted, err := s.mountUtil.IsMounted(ctx, req.TargetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "check mount: %v", err) + } + if mounted { + if err := s.mountUtil.Unmount(ctx, req.TargetPath); err != nil { + return nil, status.Errorf(codes.Internal, "unmount: %v", err) + } + } + + // CSI spec: remove mount point at target path. + os.RemoveAll(req.TargetPath) + + s.logger.Printf("NodeUnpublishVolume: %s unpublished from %s", req.VolumeId, req.TargetPath) + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: []*csi.NodeServiceCapability{ + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, + }, + }, nil +} + +func (s *nodeServer) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + return &csi.NodeGetInfoResponse{ + NodeId: s.nodeID, + MaxVolumesPerNode: 256, + AccessibleTopology: &csi.Topology{ + Segments: map[string]string{ + fmt.Sprintf("topology.%s/node", DriverName): s.nodeID, + }, + }, + }, nil +} diff --git a/weed/storage/blockvol/csi/node_test.go b/weed/storage/blockvol/csi/node_test.go new file mode 100644 index 000000000..802ab56e3 --- /dev/null +++ b/weed/storage/blockvol/csi/node_test.go @@ -0,0 +1,451 @@ +package csi + +import ( + "context" + "errors" + "log" + "os" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newTestNodeServer(t *testing.T) (*nodeServer, *mockISCSIUtil, *mockMountUtil) { + t.Helper() + mgr := newTestManager(t) + + // Pre-create a volume for stage/unstage tests. + if err := mgr.CreateVolume("test-vol", 4*1024*1024); err != nil { + t.Fatalf("create test-vol: %v", err) + } + // Close it so OpenVolume in NodeStageVolume can reopen it. + if err := mgr.CloseVolume("test-vol"); err != nil { + t.Fatalf("close test-vol: %v", err) + } + + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sda" + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: mgr, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: make(map[string]*stagedVolumeInfo), + } + return ns, mi, mm +} + +func TestNode_StageUnstage(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // Stage + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + }, + }) + if err != nil { + t.Fatalf("NodeStageVolume: %v", err) + } + + // Verify calls. + if len(mi.calls) < 3 { + t.Fatalf("expected at least 3 iscsi calls, got %d: %v", len(mi.calls), mi.calls) + } + if mi.calls[0] != "discovery:"+ns.mgr.ListenAddr() { + t.Fatalf("expected discovery call, got %q", mi.calls[0]) + } + + if len(mm.calls) < 1 { + t.Fatalf("expected at least 1 mount call, got %d: %v", len(mm.calls), mm.calls) + } + + // Verify staged map entry. + ns.stagedMu.Lock() + info, ok := ns.staged["test-vol"] + ns.stagedMu.Unlock() + if !ok { + t.Fatal("expected test-vol in staged map") + } + if !info.isLocal { + t.Fatal("expected isLocal=true for local volume") + } + + // Unstage + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("NodeUnstageVolume: %v", err) + } + + // Verify staged map cleared. + ns.stagedMu.Lock() + _, ok = ns.staged["test-vol"] + ns.stagedMu.Unlock() + if ok { + t.Fatal("expected test-vol removed from staged map") + } +} + +func TestNode_PublishUnpublish(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + stagingPath := t.TempDir() + targetPath := t.TempDir() + + _, err := ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + TargetPath: targetPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + }, + }) + if err != nil { + t.Fatalf("NodePublishVolume: %v", err) + } + + _, err = ns.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: "test-vol", + TargetPath: targetPath, + }) + if err != nil { + t.Fatalf("NodeUnpublishVolume: %v", err) + } +} + +func TestNode_StageIdempotent(t *testing.T) { + ns, _, mm := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // Mark as already mounted -> stage should be idempotent. + mm.isMountedTargets[stagingPath] = true + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err != nil { + t.Fatalf("idempotent NodeStageVolume: %v", err) + } + + // No iscsi or mount calls should have been made. + if len(mm.calls) != 0 { + t.Fatalf("expected 0 mount calls, got %d: %v", len(mm.calls), mm.calls) + } +} + +func TestNode_StageLoginFailure(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + + mi.loginErr = errors.New("connection refused") + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Fatal("expected error from login failure") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.Internal { + t.Fatalf("expected Internal error, got: %v", err) + } +} + +func TestNode_StageMkfsFailure(t *testing.T) { + ns, _, mm := newTestNodeServer(t) + + mm.formatAndMountErr = errors.New("mkfs failed") + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Fatal("expected error from mkfs failure") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.Internal { + t.Fatalf("expected Internal error, got: %v", err) + } +} + +// TestNode_StageLoginFailureCleanup verifies that the volume is closed/disconnected +// when login fails during NodeStageVolume (Finding #3: resource leak). +func TestNode_StageLoginFailureCleanup(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + + mi.loginErr = errors.New("connection refused") + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Fatal("expected error from login failure") + } + + // Volume should have been cleaned up (closed) after error. + if ns.mgr.VolumeExists("test-vol") { + t.Fatal("expected volume to be closed after login failure (resource leak)") + } +} + +// TestNode_PublishMissingStagingPath verifies that NodePublishVolume rejects +// empty StagingTargetPath (Finding #2). +func TestNode_PublishMissingStagingPath(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + _, err := ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: "", + TargetPath: t.TempDir(), + }) + if err == nil { + t.Fatal("expected error for empty StagingTargetPath") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument error, got: %v", err) + } +} + +// --- Remote target tests --- + +// TestNode_StageRemoteTarget verifies NodeStageVolume reads iSCSI info from volume_context +// instead of using local VolumeManager. +func TestNode_StageRemoteTarget(t *testing.T) { + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sdb" + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, // no local manager + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: make(map[string]*stagedVolumeInfo), + } + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "remote-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: map[string]string{ + "iscsiAddr": "10.0.0.5:3260", + "iqn": "iqn.2024.com.seaweedfs:remote-vol", + }, + }) + if err != nil { + t.Fatalf("NodeStageVolume: %v", err) + } + + // Verify discovery was called with remote portal. + if len(mi.calls) < 1 || mi.calls[0] != "discovery:10.0.0.5:3260" { + t.Fatalf("expected discovery with remote portal, got: %v", mi.calls) + } + + // Verify staged map has remote info. + ns.stagedMu.Lock() + info, ok := ns.staged["remote-vol"] + ns.stagedMu.Unlock() + if !ok { + t.Fatal("expected remote-vol in staged map") + } + if info.isLocal { + t.Fatal("expected isLocal=false for remote volume") + } + if info.iqn != "iqn.2024.com.seaweedfs:remote-vol" { + t.Fatalf("unexpected IQN: %s", info.iqn) + } +} + +// TestNode_UnstageRemoteTarget verifies unstage uses the staged map IQN. +func TestNode_UnstageRemoteTarget(t *testing.T) { + mi := newMockISCSIUtil() + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: map[string]*stagedVolumeInfo{ + "remote-vol": { + iqn: "iqn.2024.com.seaweedfs:remote-vol", + iscsiAddr: "10.0.0.5:3260", + isLocal: false, + }, + }, + } + + stagingPath := t.TempDir() + + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "remote-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("NodeUnstageVolume: %v", err) + } + + // Verify logout was called with correct IQN. + foundLogout := false + for _, c := range mi.calls { + if c == "logout:iqn.2024.com.seaweedfs:remote-vol" { + foundLogout = true + } + } + if !foundLogout { + t.Fatalf("expected logout call with remote IQN, got: %v", mi.calls) + } +} + +// TestNode_UnstageAfterRestart verifies IQN derivation when staged map is empty. +func TestNode_UnstageAfterRestart(t *testing.T) { + mi := newMockISCSIUtil() + mm := newMockMountUtil() + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: make(map[string]*stagedVolumeInfo), // empty (simulates restart) + } + + stagingPath := t.TempDir() + + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "restart-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("NodeUnstageVolume: %v", err) + } + + // Verify logout was called with derived IQN. + foundLogout := false + for _, c := range mi.calls { + if c == "logout:iqn.2024.com.seaweedfs:restart-vol" { + foundLogout = true + } + } + if !foundLogout { + t.Fatalf("expected logout with derived IQN, got: %v", mi.calls) + } +} + +// TestNode_UnstageRetryKeepsStagedEntry verifies that if unmount fails, +// the staged entry is preserved for correct retry behavior. +func TestNode_UnstageRetryKeepsStagedEntry(t *testing.T) { + mi := newMockISCSIUtil() + mm := newMockMountUtil() + mm.unmountErr = errors.New("device busy") + + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node-1", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-node] ", log.LstdFlags), + staged: map[string]*stagedVolumeInfo{ + "busy-vol": { + iqn: "iqn.2024.com.seaweedfs:busy-vol", + iscsiAddr: "10.0.0.5:3260", + isLocal: false, + }, + }, + } + + stagingPath := t.TempDir() + + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "busy-vol", + StagingTargetPath: stagingPath, + }) + if err == nil { + t.Fatal("expected error from unmount failure") + } + + // Staged entry should still be present for retry. + ns.stagedMu.Lock() + info, ok := ns.staged["busy-vol"] + ns.stagedMu.Unlock() + if !ok { + t.Fatal("staged entry should be preserved on failure for retry") + } + if info.iqn != "iqn.2024.com.seaweedfs:busy-vol" { + t.Fatalf("unexpected IQN: %s", info.iqn) + } +} + +// TestNode_StageFallbackLocal verifies local fallback when no volume_context is provided. +func TestNode_StageFallbackLocal(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // Stage without volume_context — should use local VolumeManager. + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err != nil { + t.Fatalf("NodeStageVolume: %v", err) + } + + // Should have called discovery with local addr. + if len(mi.calls) < 1 { + t.Fatal("expected iSCSI calls") + } + if mi.calls[0] != "discovery:"+ns.mgr.ListenAddr() { + t.Fatalf("expected local discovery, got %q", mi.calls[0]) + } + + // Verify staged as local. + ns.stagedMu.Lock() + info := ns.staged["test-vol"] + ns.stagedMu.Unlock() + if info == nil || !info.isLocal { + t.Fatal("expected isLocal=true for local fallback") + } +} diff --git a/weed/storage/blockvol/csi/qa_cp62_test.go b/weed/storage/blockvol/csi/qa_cp62_test.go new file mode 100644 index 000000000..fa5a640aa --- /dev/null +++ b/weed/storage/blockvol/csi/qa_cp62_test.go @@ -0,0 +1,997 @@ +package csi + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ============================================================ +// QA-NODE-CP62: Node adversarial tests for remote + staged map +// ============================================================ + +// QA-NODE-R1: Stage a remote target then unstage — staged map must track isLocal=false. +// Unstage should NOT call CloseVolume (remote volumes aren't managed locally). +func TestQA_Node_RemoteUnstageNoCloseVolume(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + mi.getDeviceResult = "/dev/sdb" + + stagingPath := t.TempDir() + + // Stage with remote volume_context (not using local mgr). + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "remote-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: map[string]string{ + "iscsiAddr": "10.0.0.5:3260", + "iqn": "iqn.2024.com.seaweedfs:remote-vol", + }, + }) + if err != nil { + t.Fatalf("stage remote: %v", err) + } + + // Verify staged entry has isLocal=false. + ns.stagedMu.Lock() + info := ns.staged["remote-vol"] + ns.stagedMu.Unlock() + if info == nil { + t.Fatal("remote-vol not in staged map") + } + if info.isLocal { + t.Fatal("BUG: remote volume should have isLocal=false") + } + + // Unstage. + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "remote-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("unstage: %v", err) + } + + // Verify CloseVolume was NOT called (remote volume). + for _, call := range mm.calls { + if strings.Contains(call, "close") { + t.Error("BUG: CloseVolume should not be called for remote volumes") + } + } + + // Verify staged entry is removed after successful unstage. + ns.stagedMu.Lock() + if _, ok := ns.staged["remote-vol"]; ok { + t.Error("BUG: staged entry should be removed after successful unstage") + } + ns.stagedMu.Unlock() +} + +// QA-NODE-R2: Stage remote, then unstage fails (unmount error) — staged entry preserved. +func TestQA_Node_RemoteUnstageFailPreservesStaged(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + mi.getDeviceResult = "/dev/sdb" + + stagingPath := t.TempDir() + + // Stage remote. + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "fail-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: map[string]string{ + "iscsiAddr": "10.0.0.5:3260", + "iqn": "iqn.2024.com.seaweedfs:fail-vol", + }, + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + + // Make unmount fail. + mm.unmountErr = errors.New("device busy") + + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "fail-vol", + StagingTargetPath: stagingPath, + }) + if err == nil { + t.Fatal("expected error from unmount failure") + } + + // Staged entry should still be present (for retry). + ns.stagedMu.Lock() + info := ns.staged["fail-vol"] + ns.stagedMu.Unlock() + if info == nil { + t.Fatal("BUG: staged entry removed despite unstage failure") + } + if info.iqn != "iqn.2024.com.seaweedfs:fail-vol" { + t.Fatalf("staged IQN mismatch: %q", info.iqn) + } +} + +// QA-NODE-R3: Concurrent Stage and Unstage for the same volumeID. +func TestQA_Node_ConcurrentStageUnstage(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + mi.getDeviceResult = "/dev/sdb" + _ = mm + + var wg sync.WaitGroup + var panicked atomic.Bool + + for i := 0; i < 20; i++ { + wg.Add(2) + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC in stage: %v", r) + } + }() + ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: t.TempDir(), + VolumeCapability: testVolCap(), + }) + }() + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC in unstage: %v", r) + } + }() + ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: "/tmp/staging", + }) + }() + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent stage/unstage caused panic") + } +} + +// QA-NODE-R4: Stage with remote volume_context should use the remote portal, not local. +// Verify iSCSI discovery and login calls use the correct portal from volume_context. +func TestQA_Node_RemotePortalUsedCorrectly(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + mi.getDeviceResult = "/dev/sdc" + + stagingPath := t.TempDir() + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "remote-portal-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: map[string]string{ + "iscsiAddr": "192.168.1.100:3260", + "iqn": "iqn.2024.com.seaweedfs:remote-portal-vol", + }, + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + + // Verify discovery was called with the REMOTE portal, not local. + foundDiscovery := false + for _, call := range mi.calls { + if strings.Contains(call, "discovery:192.168.1.100:3260") { + foundDiscovery = true + } + if strings.Contains(call, "discovery:127.0.0.1") { + t.Error("BUG: discovery used local portal instead of remote") + } + } + if !foundDiscovery { + t.Error("discovery not called with remote portal 192.168.1.100:3260") + } +} + +// QA-NODE-R5: Stage with partial volume_context (only iscsiAddr, no iqn) should fallback to local. +func TestQA_Node_PartialVolumeContext(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + mi.getDeviceResult = "/dev/sda" + + stagingPath := t.TempDir() + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: map[string]string{ + "iscsiAddr": "10.0.0.5:3260", + // Missing "iqn" — should fallback to local. + }, + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + + // Should have used local mgr (isLocal=true). + ns.stagedMu.Lock() + info := ns.staged["test-vol"] + ns.stagedMu.Unlock() + if info == nil { + t.Fatal("not in staged map") + } + if !info.isLocal { + t.Error("BUG: partial volume_context should fallback to local") + } +} + +// QA-NODE-R6: Unstage after restart with no mgr and no iqnPrefix — IQN derivation fails gracefully. +func TestQA_Node_UnstageNoMgrNoPrefix(t *testing.T) { + mi := newMockISCSIUtil() + mm := newMockMountUtil() + ns := &nodeServer{ + mgr: nil, + nodeID: "test-node", + iqnPrefix: "", // no prefix + iscsiUtil: mi, + mountUtil: mm, + logger: log.New(os.Stderr, "[test-qa] ", log.LstdFlags), + staged: make(map[string]*stagedVolumeInfo), + } + + // Unstage with no staged info, no mgr, no prefix — should still succeed + // (logout is skipped because IQN is empty). + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "ghost-vol", + StagingTargetPath: t.TempDir(), + }) + if err != nil { + t.Fatalf("unstage should succeed gracefully: %v", err) + } + + // Verify logout was NOT called (no IQN to logout from). + for _, call := range mi.calls { + if strings.Contains(call, "logout") { + t.Error("BUG: logout called without IQN") + } + } +} + +// ============================================================ +// QA-CTRL-CP62: Controller adversarial tests with VolumeBackend +// ============================================================ + +// QA-CTRL-B1: CreateVolume returns volume_context with iSCSI info. +func TestQA_Ctrl_VolumeContextPresent(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "ctx-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + if resp.Volume.VolumeContext == nil { + t.Fatal("BUG: volume_context is nil") + } + if resp.Volume.VolumeContext["iqn"] == "" { + t.Error("BUG: volume_context missing 'iqn'") + } + if resp.Volume.VolumeContext["iscsiAddr"] == "" { + t.Error("BUG: volume_context missing 'iscsiAddr'") + } +} + +// QA-CTRL-B2: ValidateVolumeCapabilities via backend — should use LookupVolume. +func TestQA_Ctrl_ValidateUsesBackend(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + // Create a volume first. + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "validate-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Validate should succeed. + _, err = cs.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: "validate-vol", + VolumeCapabilities: []*csi.VolumeCapability{ + {AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}}, + }, + }) + if err != nil { + t.Fatalf("validate: %v", err) + } + + // Delete the volume. + _, err = cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{VolumeId: "validate-vol"}) + if err != nil { + t.Fatalf("delete: %v", err) + } + + // Validate should now fail with NotFound. + _, err = cs.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: "validate-vol", + VolumeCapabilities: []*csi.VolumeCapability{ + {AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}}, + }, + }) + if err == nil { + t.Fatal("validate should fail after delete") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got: %v", err) + } +} + +// QA-CTRL-B3: CreateVolume then CreateVolume with LARGER size — backend should reject. +func TestQA_Ctrl_CreateLargerSizeRejected(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "grow-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Second create with larger size. + _, err = cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "grow-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 8 * 1024 * 1024}, + }) + if err == nil { + t.Fatal("expected error for larger size") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.AlreadyExists { + t.Fatalf("expected AlreadyExists, got: %v", err) + } +} + +// QA-CTRL-B4: CreateVolume with RequiredBytes exactly at blockSize boundary — no rounding needed. +func TestQA_Ctrl_ExactBlockSizeBoundary(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "exact-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, // exactly 4 MiB, aligned to 4096 + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if resp.Volume.CapacityBytes != 4*1024*1024 { + t.Errorf("capacity: got %d, want %d", resp.Volume.CapacityBytes, 4*1024*1024) + } +} + +// QA-CTRL-B5: Concurrent CreateVolume calls for same name via backend. +func TestQA_Ctrl_ConcurrentCreate(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + var wg sync.WaitGroup + var panicked atomic.Bool + errors := make([]error, 10) + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC: %v", r) + } + }() + _, errors[i] = cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "concurrent-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + }(i) + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent CreateVolume caused panic") + } + + // All should succeed (idempotent). + for i, err := range errors { + if err != nil { + t.Errorf("goroutine %d: %v", i, err) + } + } +} + +// ============================================================ +// QA-BACKEND: LocalVolumeBackend adversarial tests +// ============================================================ + +// QA-BACKEND-1: LookupVolume for a volume that exists on disk but not in-memory (after restart). +func TestQA_Backend_LookupAfterRestart(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + + // Phase 1: create volume, stop manager. + mgr1 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + mgr1.Start(context.Background()) + mgr1.CreateVolume("orphan-vol", 4*1024*1024) + mgr1.Stop() + + // Phase 2: new manager — volume exists on disk but not tracked. + mgr2 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + mgr2.Start(context.Background()) + defer mgr2.Stop() + + backend := NewLocalVolumeBackend(mgr2) + + // LookupVolume should fail (not tracked). + _, err := backend.LookupVolume(context.Background(), "orphan-vol") + if err == nil { + t.Fatal("BUG: LookupVolume should fail for untracked volume") + } + + // CreateVolume should re-adopt from disk. + info, err := backend.CreateVolume(context.Background(), "orphan-vol", 4*1024*1024) + if err != nil { + t.Fatalf("re-adopt: %v", err) + } + if info.CapacityBytes < 4*1024*1024 { + t.Fatalf("capacity: got %d, want >= %d", info.CapacityBytes, 4*1024*1024) + } + + // Now lookup should succeed. + _, err = backend.LookupVolume(context.Background(), "orphan-vol") + if err != nil { + t.Fatalf("lookup after re-adopt: %v", err) + } +} + +// QA-BACKEND-2: DeleteVolume then LookupVolume — should fail. +func TestQA_Backend_DeleteThenLookup(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + _, err := backend.CreateVolume(context.Background(), "del-vol", 4*1024*1024) + if err != nil { + t.Fatalf("create: %v", err) + } + + err = backend.DeleteVolume(context.Background(), "del-vol") + if err != nil { + t.Fatalf("delete: %v", err) + } + + _, err = backend.LookupVolume(context.Background(), "del-vol") + if err == nil { + t.Fatal("BUG: LookupVolume should fail after delete") + } +} + +// ============================================================ +// QA-NAMING: Cross-layer naming consistency +// ============================================================ + +// QA-NAMING-1: Verify IQN generated by VolumeManager matches IQN generated by BlockService. +// Both should use blockvol.SanitizeIQN. +func TestQA_Naming_CrossLayerConsistency(t *testing.T) { + testNames := []string{ + "pvc-abc-123", + "VolA", + "has spaces", + "UPPER_CASE", + "special!@#$chars", + strings.Repeat("long-name-", 10), // 100 chars, triggers truncation + } + + for _, name := range testNames { + // What VolumeManager would generate. + vmIQN := "iqn.2024.com.seaweedfs:" + blockvol.SanitizeIQN(name) + + // What BlockService would generate (same prefix + SanitizeIQN). + bsIQN := "iqn.2024.com.seaweedfs:" + blockvol.SanitizeIQN(name) + + if vmIQN != bsIQN { + t.Errorf("IQN mismatch for %q: VM=%q, BS=%q", name, vmIQN, bsIQN) + } + + // Filename consistency. + vmFile := blockvol.SanitizeFilename(name) + ".blk" + bsFile := blockvol.SanitizeFilename(name) + ".blk" + if vmFile != bsFile { + t.Errorf("filename mismatch for %q: VM=%q, BS=%q", name, vmFile, bsFile) + } + } +} + +// QA-NAMING-2: Two different names that produce the same sanitized IQN. +// This shouldn't happen for typical CSI volume IDs, but test the hash suffix behavior. +func TestQA_Naming_LongNameHashCollision(t *testing.T) { + // Two names that are identical for the first 55 chars but differ at the end. + // Both exceed 64 chars, so they get truncated with hash suffix. + name1 := strings.Repeat("a", 55) + "-suffix-one" + name2 := strings.Repeat("a", 55) + "-suffix-two" + + iqn1 := blockvol.SanitizeIQN(name1) + iqn2 := blockvol.SanitizeIQN(name2) + + if iqn1 == iqn2 { + t.Errorf("BUG: different names produced same IQN (hash collision):\n name1=%q\n name2=%q\n iqn=%q", name1, name2, iqn1) + } + + // Both should be <= 64 chars. + if len(iqn1) > 64 { + t.Errorf("iqn1 too long: %d", len(iqn1)) + } + if len(iqn2) > 64 { + t.Errorf("iqn2 too long: %d", len(iqn2)) + } +} + +// ============================================================ +// QA-LIFECYCLE: End-to-end lifecycle with remote targets +// ============================================================ + +// QA-LIFECYCLE-1: Full remote lifecycle: create → stage (remote) → publish → unpublish → unstage → delete. +func TestQA_RemoteLifecycleFull(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger) + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + defer mgr.Stop() + + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sda" + mm := newMockMountUtil() + + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + ns := &nodeServer{ + mgr: nil, // simulate node-only mode (no local mgr) + nodeID: "test-node", + iqnPrefix: "iqn.2024.com.seaweedfs", + iscsiUtil: mi, + mountUtil: mm, + logger: logger, + staged: make(map[string]*stagedVolumeInfo), + } + + // Create volume via controller (this creates it locally, but we'll use it as if remote). + createResp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "remote-life-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + // Stage using volume_context (simulating remote node). + stagingPath := filepath.Join(t.TempDir(), "staging") + targetPath := filepath.Join(t.TempDir(), "target") + + _, err = ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "remote-life-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + VolumeContext: createResp.Volume.VolumeContext, + }) + if err != nil { + t.Fatalf("stage: %v", err) + } + + // Publish. + _, err = ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: "remote-life-vol", + StagingTargetPath: stagingPath, + TargetPath: targetPath, + }) + if err != nil { + t.Fatalf("publish: %v", err) + } + + // Unpublish. + _, err = ns.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: "remote-life-vol", + TargetPath: targetPath, + }) + if err != nil { + t.Fatalf("unpublish: %v", err) + } + + // Unstage. + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "remote-life-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("unstage: %v", err) + } + + // Delete. + _, err = cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{ + VolumeId: "remote-life-vol", + }) + if err != nil { + t.Fatalf("delete: %v", err) + } + + // Verify file is gone. + volPath := filepath.Join(dir, sanitizeFilename("remote-life-vol")+".blk") + if _, statErr := os.Stat(volPath); !os.IsNotExist(statErr) { + t.Errorf(".blk file not cleaned up: %v", statErr) + } +} + +// QA-LIFECYCLE-2: Mode validation — controller mode should not need local VolumeManager. +func TestQA_ModeControllerNoMgr(t *testing.T) { + // In controller mode with masterAddr, no local VolumeManager is needed. + // We can't test MasterVolumeClient without a real master, but we can verify + // that the driver setup logic is correct. + _, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test.sock", + DataDir: t.TempDir(), + NodeID: "test-node", + MasterAddr: "master:9333", + Mode: "controller", + }) + if err != nil { + // MasterVolumeClient creation should work (connection is lazy). + // If this fails, it means the driver config validation is wrong. + t.Logf("NewCSIDriver controller mode: %v (may fail without grpc deps, OK)", err) + } +} + +// QA-LIFECYCLE-3: Driver with mode "node" should not create controller service. +func TestQA_ModeNodeOnly(t *testing.T) { + driver, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test-node.sock", + DataDir: t.TempDir(), + NodeID: "test-node", + Mode: "node", + }) + if err != nil { + t.Fatalf("NewCSIDriver node mode: %v", err) + } + + // In node mode, controller should be nil. + if driver.controller != nil { + t.Error("BUG: controller should be nil in node mode") + } + // Node should be non-nil. + if driver.node == nil { + t.Error("BUG: node should be non-nil in node mode") + } +} + +// QA-LIFECYCLE-4: Driver with invalid mode should error. +func TestQA_ModeInvalid(t *testing.T) { + _, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test.sock", + DataDir: t.TempDir(), + NodeID: "test-node", + Mode: "invalid", + }) + if err == nil { + t.Fatal("expected error for invalid mode") + } +} + +// ============================================================ +// QA-SERVER: Server/Driver configuration adversarial tests +// ============================================================ + +// QA-SRV-CP62-1: DriverConfig with mode="all" and no masterAddr uses local backend. +func TestQA_Srv_AllModeLocalBackend(t *testing.T) { + driver, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test-all.sock", + DataDir: t.TempDir(), + NodeID: "test-node", + Mode: "all", + }) + if err != nil { + t.Fatalf("NewCSIDriver: %v", err) + } + if driver.controller == nil { + t.Error("controller should be non-nil in 'all' mode") + } + if driver.node == nil { + t.Error("node should be non-nil in 'all' mode") + } + if driver.mgr == nil { + t.Error("mgr should be non-nil when no masterAddr and mode is 'all'") + } +} + +// QA-SRV-CP62-2: Multiple calls to driver.Stop() should not panic. +func TestQA_Srv_DoubleStop(t *testing.T) { + driver, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test-stop.sock", + DataDir: t.TempDir(), + NodeID: "test-node", + Mode: "all", + }) + if err != nil { + t.Fatalf("NewCSIDriver: %v", err) + } + + defer func() { + if r := recover(); r != nil { + t.Fatalf("BUG: double Stop() panicked: %v", r) + } + }() + + driver.Stop() + driver.Stop() // should not panic +} + +// ============================================================ +// QA-VM-CP62: VolumeManager adversarial tests (CP6-2 additions) +// ============================================================ + +// QA-VM-CP62-1: CreateVolume after Stop without Start — should return ErrNotReady. +func TestQA_VM_CreateAfterStop(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + mgr.Start(context.Background()) + mgr.Stop() + + err := mgr.CreateVolume("vol1", 4*1024*1024) + if !errors.Is(err, ErrNotReady) { + t.Fatalf("expected ErrNotReady after Stop, got: %v", err) + } +} + +// QA-VM-CP62-2: OpenVolume on a volume that doesn't exist on disk. +func TestQA_VM_OpenNonExistent(t *testing.T) { + mgr := newTestManager(t) + err := mgr.OpenVolume("does-not-exist") + if err == nil { + t.Fatal("OpenVolume for non-existent should fail") + } +} + +// QA-VM-CP62-3: ListenAddr returns empty string after Stop. +func TestQA_VM_ListenAddrAfterStop(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + mgr.Start(context.Background()) + + addr := mgr.ListenAddr() + if addr == "" { + t.Fatal("ListenAddr should be non-empty while running") + } + + mgr.Stop() + + addr = mgr.ListenAddr() + if addr != "" { + t.Logf("ListenAddr after Stop: %q (may return stale addr, not a bug if documented)", addr) + } +} + +// QA-VM-CP62-4: VolumeIQN uses shared sanitization. +func TestQA_VM_VolumeIQNSanitized(t *testing.T) { + mgr := newTestManager(t) + + iqn := mgr.VolumeIQN("pvc-ABC/def:123") + expected := "iqn.2024.com.seaweedfs:" + blockvol.SanitizeIQN("pvc-ABC/def:123") + if iqn != expected { + t.Errorf("VolumeIQN: got %q, want %q", iqn, expected) + } + + // Should be lowercase. + if strings.ToLower(iqn) != iqn { + t.Errorf("VolumeIQN not fully lowercase: %q", iqn) + } +} + +// ============================================================ +// QA-EDGE: Edge case tests +// ============================================================ + +// QA-EDGE-1: CreateVolume with RequiredBytes = minVolumeSizeBytes (1 MiB) exactly. +func TestQA_Edge_MinSize(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "min-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 1 << 20}, // exactly 1 MiB + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if resp.Volume.CapacityBytes < 1<<20 { + t.Errorf("capacity too small: got %d, want >= %d", resp.Volume.CapacityBytes, 1<<20) + } +} + +// QA-EDGE-2: CreateVolume with RequiredBytes just below minVolumeSizeBytes. +func TestQA_Edge_BelowMinSize(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "sub-min-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 100}, // 100 bytes, below min + }) + if err != nil { + t.Fatalf("create: %v", err) + } + // Should be rounded up to minVolumeSizeBytes. + if resp.Volume.CapacityBytes < 1<<20 { + t.Errorf("capacity: got %d, expected >= minVolumeSizeBytes (1 MiB)", resp.Volume.CapacityBytes) + } +} + +// QA-EDGE-3: CreateVolume with RequiredBytes = LimitBytes (exactly equal). +func TestQA_Edge_RequiredEqualsLimit(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + size := int64(4 * 1024 * 1024) + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "exact-limit-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: size, + LimitBytes: size, + }, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + if resp.Volume.CapacityBytes != size { + t.Errorf("capacity: got %d, want %d", resp.Volume.CapacityBytes, size) + } +} + +// QA-EDGE-4: CreateVolume with RequiredBytes = 4097 (needs rounding to 8192). +// With LimitBytes = 4097, the rounded size exceeds LimitBytes — should fail. +func TestQA_Edge_RoundingExceedsLimit(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "round-exceed", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 4097, + LimitBytes: 4097, + }, + }) + // RequiredBytes=4097, rounds to 8192, but LimitBytes=4097 < 8192. + // However, since RequiredBytes < minVolumeSizeBytes (1 MiB), it gets bumped to 1 MiB. + // LimitBytes=4097 < 1 MiB, so sizeBytes (1 MiB) > LimitBytes → should fail. + if err == nil { + t.Fatal("expected error: rounded size exceeds LimitBytes") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got: %v", err) + } +} + +// QA-EDGE-5: Empty string volume name in node operations. +func TestQA_Edge_EmptyVolumeIDNode(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "", + StagingTargetPath: "/tmp/staging", + }) + if err == nil { + t.Error("stage with empty volumeID should fail") + } + + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "", + StagingTargetPath: "/tmp/staging", + }) + if err == nil { + t.Error("unstage with empty volumeID should fail") + } + + _, err = ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: "", + TargetPath: "/tmp/target", + }) + if err == nil { + t.Error("publish with empty volumeID should fail") + } + + _, err = ns.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: "", + TargetPath: "/tmp/target", + }) + if err == nil { + t.Error("unpublish with empty volumeID should fail") + } +} + +// QA-EDGE-6: Sanitization: volume name with only dots. +func TestQA_Edge_AllDotsName(t *testing.T) { + name := "..." + iqn := blockvol.SanitizeIQN(name) + file := blockvol.SanitizeFilename(name) + + if iqn == "" { + t.Error("SanitizeIQN('...') should not be empty") + } + if file == "" { + t.Error("SanitizeFilename('...') should not be empty") + } + t.Logf("SanitizeIQN('...')=%q, SanitizeFilename('...')=%q", iqn, file) + + // The filename "..." -> "..." (dots are valid). Check it doesn't create + // a directory traversal. + if strings.Contains(file, "..") { + t.Logf("WARNING: sanitized filename contains '..': %q (could be path traversal in filepath.Join)", file) + } +} + +// QA-EDGE-7: Large number of volumes registered, then full heartbeat reconcile. +func TestQA_Edge_LargeScaleHeartbeat(t *testing.T) { + mgr := newTestManager(t) + + // Create 100 volumes. + for i := 0; i < 100; i++ { + name := fmt.Sprintf("scale-vol-%d", i) + if err := mgr.CreateVolume(name, 1<<20); err != nil { // 1 MiB each + t.Fatalf("create %s: %v", name, err) + } + } + + // Verify all exist. + for i := 0; i < 100; i++ { + if !mgr.VolumeExists(fmt.Sprintf("scale-vol-%d", i)) { + t.Fatalf("scale-vol-%d not found", i) + } + } + + // Delete all. + for i := 0; i < 100; i++ { + mgr.DeleteVolume(fmt.Sprintf("scale-vol-%d", i)) + } + + // Verify all gone. + for i := 0; i < 100; i++ { + if mgr.VolumeExists(fmt.Sprintf("scale-vol-%d", i)) { + t.Fatalf("scale-vol-%d still exists after delete", i) + } + } +} diff --git a/weed/storage/blockvol/csi/qa_csi_test.go b/weed/storage/blockvol/csi/qa_csi_test.go new file mode 100644 index 000000000..fa98742ca --- /dev/null +++ b/weed/storage/blockvol/csi/qa_csi_test.go @@ -0,0 +1,819 @@ +package csi + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// --- VolumeManager adversarial tests --- + +// QA-VM-1: DeleteVolume leaks snapshot delta files (.snap.N). +// CreateVolume then call BlockVol's CreateSnapshot, then DeleteVolume. +// After delete, .snap.* files should not remain on disk. +func TestQA_VM_DeleteLeaksSnapshotFiles(t *testing.T) { + mgr := newTestManager(t) + + if err := mgr.CreateVolume("snap-leak", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + // Access the managed volume to create a snapshot through the engine. + mgr.mu.RLock() + mv := mgr.volumes["snap-leak"] + mgr.mu.RUnlock() + if mv == nil { + t.Fatal("volume not tracked") + } + + // Create a snapshot to generate a .snap.0 delta file. + if err := mv.vol.CreateSnapshot(0); err != nil { + t.Fatalf("create snapshot: %v", err) + } + + // Write some data so the flusher creates CoW entries in the delta. + data := make([]byte, 4096) + for i := range data { + data[i] = 0xAB + } + if err := mv.vol.WriteLBA(0, data); err != nil { + t.Fatalf("write: %v", err) + } + + // Verify .snap.0 file exists on disk. + snapPattern := filepath.Join(mgr.dataDir, "snap-leak.blk.snap.*") + matches, _ := filepath.Glob(snapPattern) + if len(matches) == 0 { + t.Skipf("no snapshot delta files created (flusher may not have CoW'd yet)") + } + t.Logf("snapshot files before delete: %v", matches) + + // Delete the volume. + if err := mgr.DeleteVolume("snap-leak"); err != nil { + t.Fatalf("delete: %v", err) + } + + // BUG: check for leaked snapshot files. + matches, _ = filepath.Glob(snapPattern) + if len(matches) > 0 { + t.Errorf("BUG: snapshot delta files leaked after DeleteVolume: %v", matches) + } + + // Also check the main .blk file is gone. + volPath := filepath.Join(mgr.dataDir, "snap-leak.blk") + if _, err := os.Stat(volPath); !os.IsNotExist(err) { + t.Errorf("BUG: .blk file still exists after delete") + } +} + +// QA-VM-2: Start is retryable after failure. +// If initial Start fails (e.g., port in use), calling Start again after the +// port is freed should succeed. +func TestQA_VM_StartNotRetryableAfterFailure(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + + // Start first manager to occupy the port. + mgr1 := NewVolumeManager(dir, "127.0.0.1:19876", "iqn.test", logger) + if err := mgr1.Start(context.Background()); err != nil { + t.Fatalf("first start: %v", err) + } + defer mgr1.Stop() + + // Second manager on same port should fail. + mgr2 := NewVolumeManager(dir, "127.0.0.1:19876", "iqn.test", logger) + err := mgr2.Start(context.Background()) + if err == nil { + mgr2.Stop() + t.Fatal("expected second start to fail (port in use)") + } + t.Logf("first start failed as expected: %v", err) + + // Free the port, then retry — should succeed now. + mgr1.Stop() + + err = mgr2.Start(context.Background()) + if err != nil { + t.Fatalf("second start should succeed after port freed: %v", err) + } + defer mgr2.Stop() + + // Manager should be fully functional. + if err := mgr2.CreateVolume("test", 4*1024*1024); err != nil { + t.Fatalf("CreateVolume after retry: %v", err) + } +} + +// QA-VM-3: Stop then re-Start should work — manager should be fully functional. +func TestQA_VM_StopThenRestart(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + + if err := mgr.CreateVolume("v1", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + mgr.Stop() + + // Start after Stop should work. + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("restart after stop: %v", err) + } + defer mgr.Stop() + + // Manager should be fully functional — create a new volume. + if err := mgr.CreateVolume("v2", 4*1024*1024); err != nil { + t.Fatalf("create after restart: %v", err) + } + if !mgr.VolumeExists("v2") { + t.Error("volume created but not tracked") + } +} + +// QA-VM-4: CreateVolume with 0 size should return clear error. +func TestQA_VM_CreateZeroSize(t *testing.T) { + mgr := newTestManager(t) + err := mgr.CreateVolume("zero", 0) + if err == nil { + t.Error("BUG: CreateVolume with 0 size should fail") + mgr.DeleteVolume("zero") + } else { + t.Logf("correctly rejected 0-size: %v", err) + } +} + +// QA-VM-5: Concurrent CreateVolume + DeleteVolume for same name. +func TestQA_VM_ConcurrentCreateDeleteSameName(t *testing.T) { + mgr := newTestManager(t) + + var wg sync.WaitGroup + var panicked atomic.Bool + + // Run 20 goroutines, half creating and half deleting the same volume. + for i := 0; i < 20; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicked.Store(true) + t.Errorf("PANIC in goroutine %d: %v", idx, r) + } + }() + if idx%2 == 0 { + mgr.CreateVolume("race-vol", 4*1024*1024) + } else { + mgr.DeleteVolume("race-vol") + } + }(i) + } + wg.Wait() + + if panicked.Load() { + t.Fatal("BUG: concurrent create/delete caused panic") + } +} + +// QA-VM-6: Filename and IQN sanitization are consistent. +// Both sanitizeFilename and SanitizeIQN lowercase, so "VolA" and "vola" +// map to the same file and same IQN — treated as the same volume (idempotent). +func TestQA_VM_SanitizationDivergence(t *testing.T) { + mgr := newTestManager(t) + + // Both sanitizers now lowercase, so "VolA" and "vola" produce: + // filename: "vola.blk" (both) + // IQN: ":vola" (both) + // This means they are the same volume — second create is idempotent. + + err1 := mgr.CreateVolume("VolA", 4*1024*1024) + if err1 != nil { + t.Fatalf("create VolA: %v", err1) + } + + // "vola" should be idempotent (same file, same IQN, same in-memory name "VolA"). + // But note: volume names are tracked as-is in the map ("VolA" != "vola"), + // so the second create goes to file "vola.blk" which is the same file as + // "VolA" -> "vola.blk". The existing-file adoption path handles this. + err2 := mgr.CreateVolume("vola", 4*1024*1024) + if err2 != nil { + t.Fatalf("create vola (should be idempotent via file adoption): %v", err2) + } + + iqn1 := mgr.VolumeIQN("VolA") + iqn2 := mgr.VolumeIQN("vola") + + if iqn1 != iqn2 { + t.Errorf("IQN mismatch: VolA=%q, vola=%q (should be identical after lowercasing)", iqn1, iqn2) + } +} + +// QA-VM-7: OpenVolume for a volume that's already open and tracked should be idempotent. +// But what if the file was modified externally between close and reopen? +func TestQA_VM_OpenAlreadyTracked(t *testing.T) { + mgr := newTestManager(t) + + if err := mgr.CreateVolume("tracked", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + // Open again while already tracked — should be idempotent (no error). + if err := mgr.OpenVolume("tracked"); err != nil { + t.Fatalf("second open (expected idempotent): %v", err) + } +} + +// QA-VM-8: DeleteVolume for untracked volume — does it clean up .blk file from disk? +func TestQA_VM_DeleteUntrackedWithFileOnDisk(t *testing.T) { + mgr := newTestManager(t) + + // Create then close (removes from tracking but keeps file on disk). + if err := mgr.CreateVolume("orphan", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + if err := mgr.CloseVolume("orphan"); err != nil { + t.Fatalf("close: %v", err) + } + if mgr.VolumeExists("orphan") { + t.Fatal("expected volume to be untracked after close") + } + + // File should still exist on disk. + volPath := filepath.Join(mgr.dataDir, "orphan.blk") + if _, err := os.Stat(volPath); err != nil { + t.Fatalf("expected .blk file to exist: %v", err) + } + + // DeleteVolume for untracked name should still clean up file. + if err := mgr.DeleteVolume("orphan"); err != nil { + t.Fatalf("delete untracked: %v", err) + } + + if _, err := os.Stat(volPath); !os.IsNotExist(err) { + t.Errorf("BUG: .blk file not cleaned up by DeleteVolume for untracked volume") + } +} + +// --- Controller adversarial tests --- + +// QA-CTRL-1: CreateVolume with LimitBytes smaller than RequiredBytes. +// CSI spec says limit_bytes is the maximum size. If set and smaller than required, +// it should be an error. +func TestQA_Ctrl_CreateLimitLessThanRequired(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "limit-vol", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 10 * 1024 * 1024, + LimitBytes: 1 * 1024 * 1024, + }, + }) + if err == nil { + mgr.DeleteVolume("limit-vol") + t.Fatal("expected CreateVolume to reject LimitBytes < RequiredBytes") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument, got: %v", err) + } + t.Logf("correctly rejected: code=%v msg=%s", st.Code(), st.Message()) +} + +// QA-CTRL-2: CreateVolume with RequiredBytes=0 and LimitBytes set. +// Should use LimitBytes as the size. +func TestQA_Ctrl_CreateOnlyLimitBytes(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + resp, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "limit-only", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 0, + LimitBytes: 2 * 1024 * 1024, + }, + }) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + // Volume should be created but size should not exceed LimitBytes. + if resp.Volume.CapacityBytes > 2*1024*1024 { + t.Errorf("BUG: volume size %d exceeds LimitBytes %d", + resp.Volume.CapacityBytes, 2*1024*1024) + } +} + +// QA-CTRL-3: CreateVolume with name containing path traversal. +func TestQA_Ctrl_CreatePathTraversal(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "../../etc/shadow", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 4 * 1024 * 1024, + }, + }) + if err != nil { + t.Logf("path traversal rejected: %v", err) + return + } + + // If it succeeded, verify the file was NOT created outside dataDir. + if _, statErr := os.Stat("../../etc/shadow.blk"); statErr == nil { + t.Fatal("BUG: path traversal created file outside data directory!") + } + + // Check it went to a sanitized name inside dataDir. + sanitized := filepath.Join(mgr.dataDir, "..-..-etc-shadow.blk") + if _, statErr := os.Stat(sanitized); statErr == nil { + t.Logf("file created with sanitized name: %s (safe)", sanitized) + } + + mgr.DeleteVolume("../../etc/shadow") +} + +// QA-CTRL-4: ValidateVolumeCapabilities after restart (not tracked in memory). +// By design, VolumeManager does not auto-discover volumes on startup. +// Volumes are re-tracked when kubelet re-calls CreateVolume or NodeStageVolume. +// ValidateVolumeCapabilities returns NotFound for orphaned volumes — expected. +func TestQA_Ctrl_ValidateAfterRestart(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + + // Phase 1: create volume, stop. + mgr1 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + if err := mgr1.Start(context.Background()); err != nil { + t.Fatalf("start1: %v", err) + } + if err := mgr1.CreateVolume("validate-vol", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + mgr1.Stop() + + // Phase 2: new manager (simulates restart — no auto-discovery). + mgr2 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.test", logger) + if err := mgr2.Start(context.Background()); err != nil { + t.Fatalf("start2: %v", err) + } + defer mgr2.Stop() + + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr2)} + + // ValidateVolumeCapabilities for volume that exists on disk but not in memory. + // Expected: NotFound (by design — volumes are re-tracked via CreateVolume). + _, err := cs.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: "validate-vol", + VolumeCapabilities: []*csi.VolumeCapability{ + {AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}}, + }, + }) + if err == nil { + t.Fatal("expected NotFound for volume not yet re-tracked after restart") + } + st, ok := status.FromError(err) + if !ok || st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got: %v", err) + } + t.Log("correctly returns NotFound for volume not yet re-tracked (by design)") + + // After CreateVolume re-adopts it, Validate should succeed. + if err := mgr2.CreateVolume("validate-vol", 4*1024*1024); err != nil { + t.Fatalf("re-adopt: %v", err) + } + _, err = cs.ValidateVolumeCapabilities(context.Background(), &csi.ValidateVolumeCapabilitiesRequest{ + VolumeId: "validate-vol", + VolumeCapabilities: []*csi.VolumeCapability{ + {AccessType: &csi.VolumeCapability_Mount{Mount: &csi.VolumeCapability_MountVolume{}}}, + }, + }) + if err != nil { + t.Fatalf("after re-adopt, expected success: %v", err) + } +} + +// QA-CTRL-5: CreateVolume with size that overflows uint64 after rounding. +func TestQA_Ctrl_CreateMaxSize(t *testing.T) { + mgr := newTestManager(t) + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + + // Request just under max int64 — rounding up to blockSize could overflow. + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: "huge", + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{ + RequiredBytes: 1<<63 - 1, // max int64 + }, + }) + if err == nil { + t.Error("BUG: should reject unreasonably large size or fail gracefully") + mgr.DeleteVolume("huge") + } else { + t.Logf("large size handled: %v", err) + } +} + +// --- Node adversarial tests --- + +// QA-NODE-1: NodeStageVolume for a volume that doesn't exist. +func TestQA_Node_StageNonExistentVolume(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "does-not-exist", + StagingTargetPath: t.TempDir(), + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Error("BUG: should fail for non-existent volume") + } else { + t.Logf("correctly rejected: %v", err) + } +} + +// QA-NODE-2: NodeUnstageVolume when all operations fail — should propagate first error. +func TestQA_Node_UnstageAllFail(t *testing.T) { + ns, mi, mm := newTestNodeServer(t) + + mm.unmountErr = errors.New("unmount failed") + mi.logoutErr = errors.New("logout failed") + + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: "/tmp/staging", + }) + if err == nil { + t.Error("BUG: should return error when unmount and logout both fail") + } else { + // Should report the first error (unmount). + if !strings.Contains(err.Error(), "unmount") { + t.Errorf("expected unmount error to be first, got: %v", err) + } + } +} + +// QA-NODE-3: NodePublishVolume when staging path is not actually mounted. +// This should either fail or at least warn — bind-mounting an empty dir +// could silently give the pod an empty volume. +func TestQA_Node_PublishWithoutStaging(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + stagingPath := t.TempDir() + targetPath := t.TempDir() + + // Staging path is NOT mounted. + // NodePublishVolume should either check or just mount (depends on behavior). + _, err := ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + TargetPath: targetPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{FsType: "ext4"}, + }, + }, + }) + // This likely succeeds (bind mounts empty dir) — which is wrong. + if err == nil { + t.Log("WARNING: NodePublishVolume succeeded when staging path was not mounted (bind-mounts empty dir)") + } +} + +// QA-NODE-4: NodeStageVolume idempotency doesn't verify correct volume. +// If something else is mounted at the staging path, Stage returns success +// without verifying it's our volume. +func TestQA_Node_StageWrongVolumeAtPath(t *testing.T) { + ns, _, mm := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // Pre-mark staging path as mounted (simulating another volume mounted there). + mm.isMountedTargets[stagingPath] = true + + // NodeStageVolume for a different volume — should it succeed? + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Log("WARNING: NodeStageVolume returned success because staging path was already mounted, " + + "but it could be a different volume (no verification of mount source)") + } +} + +// QA-NODE-5: Double NodeUnstageVolume — should be idempotent. +func TestQA_Node_DoubleUnstage(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + stagingPath := t.TempDir() + + // First unstage — nothing to undo, but should succeed. + _, err := ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("first unstage: %v", err) + } + + // Second unstage — should also succeed (idempotent). + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Errorf("BUG: double unstage should be idempotent: %v", err) + } +} + +// QA-NODE-6: NodeGetInfo returns correct topology and max volumes. +func TestQA_Node_GetInfo(t *testing.T) { + ns, _, _ := newTestNodeServer(t) + + resp, err := ns.NodeGetInfo(context.Background(), &csi.NodeGetInfoRequest{}) + if err != nil { + t.Fatalf("NodeGetInfo: %v", err) + } + if resp.NodeId != "test-node-1" { + t.Errorf("node_id: got %q, want %q", resp.NodeId, "test-node-1") + } + if resp.MaxVolumesPerNode <= 0 { + t.Errorf("max_volumes: got %d, want > 0", resp.MaxVolumesPerNode) + } + if resp.AccessibleTopology == nil { + t.Error("expected non-nil topology") + } +} + +// QA-NODE-7: NodeStageVolume with iSCSI discovery failure should clean up. +func TestQA_Node_StageDiscoveryFailureCleanup(t *testing.T) { + ns, mi, _ := newTestNodeServer(t) + + mi.discoveryErr = errors.New("unreachable portal") + + stagingPath := t.TempDir() + + _, err := ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: "test-vol", + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err == nil { + t.Fatal("expected error from discovery failure") + } + + // Volume should be cleaned up. + if ns.mgr.VolumeExists("test-vol") { + t.Error("BUG: volume still tracked after discovery failure (resource leak)") + } +} + +// --- Server adversarial tests --- + +// QA-SRV-1: parseEndpoint with unsupported scheme. +func TestQA_Srv_ParseEndpointBadScheme(t *testing.T) { + _, _, err := parseEndpoint("http://localhost:50051") + if err == nil { + t.Error("BUG: should reject http:// scheme") + } +} + +// QA-SRV-2: parseEndpoint with various formats. +func TestQA_Srv_ParseEndpointFormats(t *testing.T) { + tests := []struct { + input string + wantProto string + wantAddr string + wantErr bool + }{ + {"unix:///csi/csi.sock", "unix", "/csi/csi.sock", false}, + {"unix:///var/lib/kubelet/plugins/block.csi/csi.sock", "unix", "/var/lib/kubelet/plugins/block.csi/csi.sock", false}, + {"tcp://0.0.0.0:50051", "tcp", "0.0.0.0:50051", false}, + {"ftp://host/path", "", "", true}, + {"", "", "", true}, + } + for _, tt := range tests { + proto, addr, err := parseEndpoint(tt.input) + if tt.wantErr { + if err == nil { + t.Errorf("parseEndpoint(%q): expected error", tt.input) + } + continue + } + if err != nil { + t.Errorf("parseEndpoint(%q): %v", tt.input, err) + continue + } + if proto != tt.wantProto || addr != tt.wantAddr { + t.Errorf("parseEndpoint(%q): got (%q, %q), want (%q, %q)", + tt.input, proto, addr, tt.wantProto, tt.wantAddr) + } + } +} + +// QA-SRV-3: NewCSIDriver with empty NodeID should fail. +func TestQA_Srv_DriverEmptyNodeID(t *testing.T) { + _, err := NewCSIDriver(DriverConfig{ + Endpoint: "unix:///tmp/test.sock", + DataDir: t.TempDir(), + NodeID: "", + }) + if err == nil { + t.Error("BUG: should reject empty NodeID") + } +} + +// --- Identity adversarial tests --- + +// QA-ID-1: Identity methods should work with nil requests. +func TestQA_Identity_NilRequests(t *testing.T) { + s := &identityServer{} + + if _, err := s.GetPluginInfo(context.Background(), nil); err != nil { + t.Errorf("GetPluginInfo(nil): %v", err) + } + if _, err := s.GetPluginCapabilities(context.Background(), nil); err != nil { + t.Errorf("GetPluginCapabilities(nil): %v", err) + } + if _, err := s.Probe(context.Background(), nil); err != nil { + t.Errorf("Probe(nil): %v", err) + } +} + +// --- SanitizeIQN adversarial tests --- + +// QA-IQN-1: IQN with only invalid characters should not produce empty string. +func TestQA_IQN_AllInvalidChars(t *testing.T) { + iqn := SanitizeIQN("!@#$%^&*()") + if iqn == "" { + t.Error("BUG: SanitizeIQN produced empty string for all-invalid input") + } + t.Logf("SanitizeIQN('!@#$%%^&*()') = %q", iqn) +} + +// QA-IQN-2: Empty string input. +func TestQA_IQN_Empty(t *testing.T) { + iqn := SanitizeIQN("") + // Empty is technically valid but probably wrong — should the caller validate? + t.Logf("SanitizeIQN('') = %q (len=%d)", iqn, len(iqn)) +} + +// QA-IQN-3: IQN at exactly 64 chars should NOT get hash suffix. +func TestQA_IQN_ExactlyMaxLength(t *testing.T) { + name := strings.Repeat("a", 64) + iqn := SanitizeIQN(name) + if len(iqn) != 64 { + t.Errorf("expected 64 chars, got %d: %q", len(iqn), iqn) + } + // Should not have hash suffix at exactly 64. + if strings.Contains(iqn, "-") && len(name) == 64 { + // This would mean it was unnecessarily truncated. + t.Log("at-boundary: has dash but input was exactly 64 chars") + } +} + +// QA-IQN-4: IQN at 65 chars should get hash suffix. +func TestQA_IQN_OneOverMax(t *testing.T) { + name := strings.Repeat("a", 65) + iqn := SanitizeIQN(name) + if len(iqn) > 64 { + t.Errorf("expected max 64 chars, got %d", len(iqn)) + } + // Verify hash suffix is present. + parts := strings.Split(iqn, "-") + if len(parts) < 2 { + t.Errorf("expected hash suffix after truncation: %q", iqn) + } +} + +// QA-IQN-5: Two names that differ only by case should produce different IQNs +// (or we should document that case is folded). +func TestQA_IQN_CaseFolding(t *testing.T) { + iqn1 := SanitizeIQN("MyVolume") + iqn2 := SanitizeIQN("myvolume") + if iqn1 != iqn2 { + t.Errorf("case folding: %q != %q (different IQNs for same logical name)", iqn1, iqn2) + } + // This is expected — IQN lowercases. But the FILENAMES may differ. + t.Logf("SanitizeIQN('MyVolume')=%q, SanitizeIQN('myvolume')=%q", iqn1, iqn2) +} + +// --- Cross-cutting adversarial tests --- + +// QA-X-1: Full lifecycle: create -> stage -> publish -> unpublish -> unstage -> delete. +// Run twice to verify second lifecycle works. +func TestQA_FullLifecycleTwice(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-qa] ", log.LstdFlags) + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger) + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + defer mgr.Stop() + + mi := newMockISCSIUtil() + mi.getDeviceResult = "/dev/sda" + mm := newMockMountUtil() + + cs := &controllerServer{backend: NewLocalVolumeBackend(mgr)} + ns := &nodeServer{ + mgr: mgr, + nodeID: "test-node", + iscsiUtil: mi, + mountUtil: mm, + logger: logger, + } + + for round := 0; round < 2; round++ { + volName := fmt.Sprintf("lifecycle-%d", round) + t.Logf("--- round %d ---", round) + + // Create + _, err := cs.CreateVolume(context.Background(), &csi.CreateVolumeRequest{ + Name: volName, + VolumeCapabilities: testVolCaps(), + CapacityRange: &csi.CapacityRange{RequiredBytes: 4 * 1024 * 1024}, + }) + if err != nil { + t.Fatalf("round %d create: %v", round, err) + } + + // Close so stage can reopen. + mgr.CloseVolume(volName) + + stagingPath := filepath.Join(t.TempDir(), "staging") + targetPath := filepath.Join(t.TempDir(), "target") + + // Stage + _, err = ns.NodeStageVolume(context.Background(), &csi.NodeStageVolumeRequest{ + VolumeId: volName, + StagingTargetPath: stagingPath, + VolumeCapability: testVolCap(), + }) + if err != nil { + t.Fatalf("round %d stage: %v", round, err) + } + + // Publish + _, err = ns.NodePublishVolume(context.Background(), &csi.NodePublishVolumeRequest{ + VolumeId: volName, + StagingTargetPath: stagingPath, + TargetPath: targetPath, + }) + if err != nil { + t.Fatalf("round %d publish: %v", round, err) + } + + // Unpublish + _, err = ns.NodeUnpublishVolume(context.Background(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: volName, + TargetPath: targetPath, + }) + if err != nil { + t.Fatalf("round %d unpublish: %v", round, err) + } + + // Unstage + _, err = ns.NodeUnstageVolume(context.Background(), &csi.NodeUnstageVolumeRequest{ + VolumeId: volName, + StagingTargetPath: stagingPath, + }) + if err != nil { + t.Fatalf("round %d unstage: %v", round, err) + } + + // Delete + _, err = cs.DeleteVolume(context.Background(), &csi.DeleteVolumeRequest{ + VolumeId: volName, + }) + if err != nil { + t.Fatalf("round %d delete: %v", round, err) + } + + // Verify file gone. + volPath := filepath.Join(dir, sanitizeFilename(volName)+".blk") + if _, statErr := os.Stat(volPath); !os.IsNotExist(statErr) { + t.Errorf("round %d: .blk file not cleaned up: %v", round, statErr) + } + } +} diff --git a/weed/storage/blockvol/csi/server.go b/weed/storage/blockvol/csi/server.go new file mode 100644 index 000000000..80676ea9a --- /dev/null +++ b/weed/storage/blockvol/csi/server.go @@ -0,0 +1,170 @@ +package csi + +import ( + "context" + "fmt" + "log" + "net" + "net/url" + "os" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc" +) + +// DriverConfig holds configuration for the CSI driver. +type DriverConfig struct { + Endpoint string // CSI endpoint (unix:///csi/csi.sock) + DataDir string // volume data directory + ISCSIAddr string // local iSCSI target listen address + IQNPrefix string // IQN prefix for volumes + NodeID string // node identifier + Logger *log.Logger + + // Control-plane mode fields. + MasterAddr string // master address for control-plane mode (empty = local/standalone) + Mode string // "controller", "node", "all" (default "all") +} + +// CSIDriver manages the gRPC server and CSI services. +type CSIDriver struct { + identity *identityServer + controller *controllerServer + node *nodeServer + mgr *VolumeManager + server *grpc.Server + endpoint string + logger *log.Logger +} + +// NewCSIDriver creates a new CSI driver from the given configuration. +func NewCSIDriver(cfg DriverConfig) (*CSIDriver, error) { + if cfg.NodeID == "" { + return nil, fmt.Errorf("csi: node ID is required") + } + if cfg.Logger == nil { + cfg.Logger = log.Default() + } + if cfg.Mode == "" { + cfg.Mode = "all" + } + switch cfg.Mode { + case "controller", "node", "all": + // valid + default: + return nil, fmt.Errorf("csi: invalid mode %q, must be controller/node/all", cfg.Mode) + } + + d := &CSIDriver{ + identity: &identityServer{}, + endpoint: cfg.Endpoint, + logger: cfg.Logger, + } + + // Create VolumeManager for modes that need local volume management. + var mgr *VolumeManager + needsLocalMgr := cfg.Mode == "all" && cfg.MasterAddr == "" || cfg.Mode == "node" + if needsLocalMgr { + mgr = NewVolumeManager(cfg.DataDir, cfg.ISCSIAddr, cfg.IQNPrefix, cfg.Logger) + d.mgr = mgr + } + + // Create backend for controller. + var backend VolumeBackend + if cfg.Mode == "controller" || cfg.Mode == "all" { + if cfg.MasterAddr != "" { + backend = NewMasterVolumeClient(cfg.MasterAddr, nil) + } else if mgr != nil { + backend = NewLocalVolumeBackend(mgr) + } else { + return nil, fmt.Errorf("csi: controller mode requires either --master or --data-dir") + } + d.controller = &controllerServer{backend: backend} + } + + // Create node server. + if cfg.Mode == "node" || cfg.Mode == "all" { + d.node = &nodeServer{ + mgr: mgr, // may be nil in controller-only mode + nodeID: cfg.NodeID, + iqnPrefix: cfg.IQNPrefix, + iscsiUtil: &realISCSIUtil{}, + mountUtil: &realMountUtil{}, + logger: cfg.Logger, + staged: make(map[string]*stagedVolumeInfo), + } + } + + return d, nil +} + +// Run starts the volume manager and gRPC server. Blocks until Stop is called. +func (d *CSIDriver) Run() error { + if d.mgr != nil { + if err := d.mgr.Start(context.Background()); err != nil { + return fmt.Errorf("csi: start volume manager: %w", err) + } + } + + // Parse endpoint URL. + proto, addr, err := parseEndpoint(d.endpoint) + if err != nil { + return fmt.Errorf("csi: parse endpoint: %w", err) + } + + // Remove existing socket file if present. + if proto == "unix" { + os.Remove(addr) + } + + ln, err := net.Listen(proto, addr) + if err != nil { + return fmt.Errorf("csi: listen %s: %w", d.endpoint, err) + } + + d.server = grpc.NewServer() + csi.RegisterIdentityServer(d.server, d.identity) + if d.controller != nil { + csi.RegisterControllerServer(d.server, d.controller) + } + if d.node != nil { + csi.RegisterNodeServer(d.server, d.node) + } + + d.logger.Printf("CSI driver serving on %s", d.endpoint) + return d.server.Serve(ln) +} + +// Stop gracefully shuts down the gRPC server and volume manager. +func (d *CSIDriver) Stop() { + if d.server != nil { + d.server.GracefulStop() + } + if d.mgr != nil { + d.mgr.Stop() + } +} + +// parseEndpoint parses a CSI endpoint string (unix:///path or tcp://host:port). +func parseEndpoint(ep string) (string, string, error) { + if strings.HasPrefix(ep, "unix://") { + u, err := url.Parse(ep) + if err != nil { + return "", "", err + } + addr := u.Path + if u.Host != "" { + addr = u.Host + addr + } + return "unix", addr, nil + } + if strings.HasPrefix(ep, "tcp://") { + u, err := url.Parse(ep) + if err != nil { + return "", "", err + } + return "tcp", u.Host, nil + } + return "", "", fmt.Errorf("unsupported endpoint scheme: %s", ep) +} diff --git a/weed/storage/blockvol/csi/volume_backend.go b/weed/storage/blockvol/csi/volume_backend.go new file mode 100644 index 000000000..7300dfc88 --- /dev/null +++ b/weed/storage/blockvol/csi/volume_backend.go @@ -0,0 +1,131 @@ +package csi + +import ( + "context" + "fmt" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" +) + +// VolumeInfo holds volume metadata returned by the backend. +type VolumeInfo struct { + VolumeID string + ISCSIAddr string // iSCSI target address (ip:port) + IQN string // iSCSI target IQN + CapacityBytes uint64 +} + +// VolumeBackend abstracts volume lifecycle for the CSI controller. +type VolumeBackend interface { + CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) + DeleteVolume(ctx context.Context, name string) error + LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) +} + +// LocalVolumeBackend wraps VolumeManager for standalone/local mode (CP6-1). +type LocalVolumeBackend struct { + mgr *VolumeManager +} + +// NewLocalVolumeBackend creates a backend backed by the local VolumeManager. +func NewLocalVolumeBackend(mgr *VolumeManager) *LocalVolumeBackend { + return &LocalVolumeBackend{mgr: mgr} +} + +func (b *LocalVolumeBackend) CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + if err := b.mgr.CreateVolume(name, sizeBytes); err != nil { + return nil, err + } + actualSize := b.mgr.VolumeSizeBytes(name) + if actualSize == 0 { + actualSize = sizeBytes + } + return &VolumeInfo{ + VolumeID: name, + ISCSIAddr: b.mgr.ListenAddr(), + IQN: b.mgr.VolumeIQN(name), + CapacityBytes: actualSize, + }, nil +} + +func (b *LocalVolumeBackend) DeleteVolume(ctx context.Context, name string) error { + return b.mgr.DeleteVolume(name) +} + +func (b *LocalVolumeBackend) LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) { + if !b.mgr.VolumeExists(name) { + return nil, fmt.Errorf("volume %q not found", name) + } + return &VolumeInfo{ + VolumeID: name, + ISCSIAddr: b.mgr.ListenAddr(), + IQN: b.mgr.VolumeIQN(name), + CapacityBytes: b.mgr.VolumeSizeBytes(name), + }, nil +} + +// MasterVolumeClient calls master gRPC for volume operations. +type MasterVolumeClient struct { + masterAddr string + dialOpt grpc.DialOption +} + +// NewMasterVolumeClient creates a client that calls the master for volume operations. +func NewMasterVolumeClient(masterAddr string, dialOpt grpc.DialOption) *MasterVolumeClient { + return &MasterVolumeClient{ + masterAddr: masterAddr, + dialOpt: dialOpt, + } +} + +func (c *MasterVolumeClient) CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + var info *VolumeInfo + err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + resp, err := client.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: name, + SizeBytes: sizeBytes, + }) + if err != nil { + return err + } + info = &VolumeInfo{ + VolumeID: resp.VolumeId, + ISCSIAddr: resp.IscsiAddr, + IQN: resp.Iqn, + CapacityBytes: resp.CapacityBytes, + } + return nil + }) + return info, err +} + +func (c *MasterVolumeClient) DeleteVolume(ctx context.Context, name string) error { + return pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + _, err := client.DeleteBlockVolume(ctx, &master_pb.DeleteBlockVolumeRequest{ + Name: name, + }) + return err + }) +} + +func (c *MasterVolumeClient) LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) { + var info *VolumeInfo + err := pb.WithMasterClient(false, pb.ServerAddress(c.masterAddr), c.dialOpt, false, func(client master_pb.SeaweedClient) error { + resp, err := client.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{ + Name: name, + }) + if err != nil { + return err + } + info = &VolumeInfo{ + VolumeID: name, + ISCSIAddr: resp.IscsiAddr, + IQN: resp.Iqn, + CapacityBytes: resp.CapacityBytes, + } + return nil + }) + return info, err +} diff --git a/weed/storage/blockvol/csi/volume_backend_test.go b/weed/storage/blockvol/csi/volume_backend_test.go new file mode 100644 index 000000000..0870b03ec --- /dev/null +++ b/weed/storage/blockvol/csi/volume_backend_test.go @@ -0,0 +1,92 @@ +package csi + +import ( + "context" + "testing" +) + +func TestBackend_LocalCreate(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + info, err := backend.CreateVolume(context.Background(), "vol1", 4*1024*1024) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + if info.VolumeID != "vol1" { + t.Fatalf("VolumeID: got %q, want vol1", info.VolumeID) + } + if info.CapacityBytes != 4*1024*1024 { + t.Fatalf("CapacityBytes: got %d, want %d", info.CapacityBytes, 4*1024*1024) + } + if info.IQN == "" { + t.Fatal("IQN should not be empty") + } + + // Lookup should find it. + looked, err := backend.LookupVolume(context.Background(), "vol1") + if err != nil { + t.Fatalf("LookupVolume: %v", err) + } + if looked.VolumeID != "vol1" || looked.IQN != info.IQN { + t.Fatalf("LookupVolume mismatch: got %+v", looked) + } +} + +func TestBackend_LocalDelete(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + if _, err := backend.CreateVolume(context.Background(), "vol1", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + if err := backend.DeleteVolume(context.Background(), "vol1"); err != nil { + t.Fatalf("delete: %v", err) + } + + // Lookup should fail. + if _, err := backend.LookupVolume(context.Background(), "vol1"); err == nil { + t.Fatal("lookup should fail after delete") + } +} + +func TestBackend_LocalIdempotent(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + info1, err := backend.CreateVolume(context.Background(), "vol1", 4*1024*1024) + if err != nil { + t.Fatalf("first create: %v", err) + } + + // Same name + same size = idempotent. + info2, err := backend.CreateVolume(context.Background(), "vol1", 4*1024*1024) + if err != nil { + t.Fatalf("second create: %v", err) + } + + if info1.IQN != info2.IQN { + t.Fatalf("IQN mismatch: %q vs %q", info1.IQN, info2.IQN) + } +} + +func TestBackend_LocalDeleteIdempotent(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + // Deleting non-existent volume should not error. + if err := backend.DeleteVolume(context.Background(), "nonexistent"); err != nil { + t.Fatalf("delete nonexistent: %v", err) + } +} + +func TestBackend_LocalLookupNotFound(t *testing.T) { + mgr := newTestManager(t) + backend := NewLocalVolumeBackend(mgr) + + _, err := backend.LookupVolume(context.Background(), "missing") + if err == nil { + t.Fatal("lookup missing should return error") + } +} diff --git a/weed/storage/blockvol/csi/volume_manager.go b/weed/storage/blockvol/csi/volume_manager.go new file mode 100644 index 000000000..2725765eb --- /dev/null +++ b/weed/storage/blockvol/csi/volume_manager.go @@ -0,0 +1,344 @@ +package csi + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "os" + "path/filepath" + "sync" + + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" +) + +var ( + ErrNotReady = errors.New("csi: volume manager not ready") + ErrVolumeExists = errors.New("csi: volume already exists") + ErrVolumeNotFound = errors.New("csi: volume not found") +) + +// managedVolume tracks a single BlockVol instance and its iSCSI target. +type managedVolume struct { + vol *blockvol.BlockVol + path string // file path to .blk file + iqn string // target IQN for this volume + sizeBytes uint64 +} + +// managerState tracks the lifecycle of the VolumeManager. +type managerState int + +const ( + stateStopped managerState = iota // initial or after Stop() + stateStarting // Start() in progress + stateReady // running normally + stateFailed // Start() failed, retryable +) + +// VolumeManager manages multiple BlockVol instances behind a shared TargetServer. +type VolumeManager struct { + mu sync.RWMutex + dataDir string + volumes map[string]*managedVolume + target *iscsi.TargetServer + iqnPrefix string + config iscsi.TargetConfig + logger *log.Logger + state managerState + iscsiAddr string +} + +// NewVolumeManager creates a new VolumeManager. +func NewVolumeManager(dataDir, iscsiAddr, iqnPrefix string, logger *log.Logger) *VolumeManager { + if logger == nil { + logger = log.Default() + } + config := iscsi.DefaultTargetConfig() + return &VolumeManager{ + dataDir: dataDir, + volumes: make(map[string]*managedVolume), + iqnPrefix: iqnPrefix, + config: config, + logger: logger, + iscsiAddr: iscsiAddr, + } +} + +// Start initializes and starts the shared TargetServer. +// Safe to call after Stop() or after a failed Start(). Returns immediately if already running. +// The listener is created synchronously so port-in-use errors surface immediately. +func (m *VolumeManager) Start(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.state == stateReady { + return nil // already running + } + if m.state == stateStarting { + return fmt.Errorf("csi: start already in progress") + } + m.state = stateStarting + + if err := os.MkdirAll(m.dataDir, 0755); err != nil { + m.state = stateFailed + return fmt.Errorf("csi: create data dir: %w", err) + } + + m.target = iscsi.NewTargetServer(m.iscsiAddr, m.config, m.logger) + + // Create listener synchronously so bind errors are reported immediately. + ln, err := net.Listen("tcp", m.iscsiAddr) + if err != nil { + m.target = nil + m.state = stateFailed + return fmt.Errorf("csi: listen %s: %w", m.iscsiAddr, err) + } + + ts := m.target // capture for goroutine (m.target may be reset by Stop) + go func() { + if err := ts.Serve(ln); err != nil { + m.logger.Printf("target server error: %v", err) + } + }() + + m.state = stateReady + m.logger.Printf("volume manager started: dataDir=%s iscsiAddr=%s", m.dataDir, ln.Addr()) + return nil +} + +// Stop closes all volumes and the target server. After Stop, Start may be called again. +func (m *VolumeManager) Stop() error { + m.mu.Lock() + defer m.mu.Unlock() + + for name, mv := range m.volumes { + if m.target != nil { + m.target.DisconnectVolume(mv.iqn) + } + mv.vol.Close() + delete(m.volumes, name) + } + + var err error + if m.target != nil { + err = m.target.Close() + m.target = nil + } + + m.state = stateStopped + return err +} + +// ErrVolumeSizeMismatch indicates a volume exists on disk with a different size. +var ErrVolumeSizeMismatch = errors.New("csi: volume exists with different size") + +// CreateVolume creates a new BlockVol file and registers it with the target. +// Idempotent: if the .blk file already exists on disk (e.g. after driver restart), +// it is opened and tracked. Returns ErrVolumeSizeMismatch if the existing volume +// has a smaller size than requested. +func (m *VolumeManager) CreateVolume(name string, sizeBytes uint64) error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.state != stateReady { + return ErrNotReady + } + + // Already tracked in-memory. + if mv, ok := m.volumes[name]; ok { + if mv.sizeBytes >= sizeBytes { + return nil // idempotent + } + return ErrVolumeSizeMismatch + } + + volPath := m.volumePath(name) + + // Check for existing .blk file on disk (survives driver restart). + if _, statErr := os.Stat(volPath); statErr == nil { + vol, err := blockvol.OpenBlockVol(volPath) + if err != nil { + return fmt.Errorf("csi: open existing blockvol: %w", err) + } + info := vol.Info() + if info.VolumeSize < sizeBytes { + vol.Close() + return ErrVolumeSizeMismatch + } + iqn := m.volumeIQN(name) + adapter := &blockvol.BlockVolAdapter{Vol: vol, TPGID: 1} + m.target.AddVolume(iqn, adapter) + m.volumes[name] = &managedVolume{ + vol: vol, + path: volPath, + iqn: iqn, + sizeBytes: info.VolumeSize, + } + m.logger.Printf("adopted existing volume %q: %s (%d bytes)", name, iqn, info.VolumeSize) + return nil + } + + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: sizeBytes, + BlockSize: 4096, + WALSize: 64 * 1024 * 1024, + }) + if err != nil { + return fmt.Errorf("csi: create blockvol: %w", err) + } + + iqn := m.volumeIQN(name) + adapter := &blockvol.BlockVolAdapter{Vol: vol, TPGID: 1} + m.target.AddVolume(iqn, adapter) + + m.volumes[name] = &managedVolume{ + vol: vol, + path: volPath, + iqn: iqn, + sizeBytes: sizeBytes, + } + + m.logger.Printf("created volume %q: %s (%d bytes)", name, iqn, sizeBytes) + return nil +} + +// DeleteVolume closes and deletes a volume file and associated snapshot files. +func (m *VolumeManager) DeleteVolume(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + mv, ok := m.volumes[name] + if !ok { + // Idempotent: try to remove file anyway in case volume was not tracked. + volPath := m.volumePath(name) + os.Remove(volPath) + removeSnapshotFiles(volPath) + return nil + } + + if m.target != nil { + m.target.DisconnectVolume(mv.iqn) + } + mv.vol.Close() + os.Remove(mv.path) + removeSnapshotFiles(mv.path) + delete(m.volumes, name) + + m.logger.Printf("deleted volume %q", name) + return nil +} + +// removeSnapshotFiles removes any .snap.* delta files associated with a volume path. +func removeSnapshotFiles(volPath string) { + matches, _ := filepath.Glob(volPath + ".snap.*") + for _, m := range matches { + os.Remove(m) + } +} + +// OpenVolume opens an existing BlockVol file and adds it to the target. +func (m *VolumeManager) OpenVolume(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.state != stateReady { + return ErrNotReady + } + + if _, ok := m.volumes[name]; ok { + return nil // already open, idempotent + } + + volPath := m.volumePath(name) + vol, err := blockvol.OpenBlockVol(volPath) + if err != nil { + return fmt.Errorf("csi: open blockvol: %w", err) + } + + info := vol.Info() + iqn := m.volumeIQN(name) + adapter := &blockvol.BlockVolAdapter{Vol: vol, TPGID: 1} + m.target.AddVolume(iqn, adapter) + + m.volumes[name] = &managedVolume{ + vol: vol, + path: volPath, + iqn: iqn, + sizeBytes: info.VolumeSize, + } + + m.logger.Printf("opened volume %q: %s", name, iqn) + return nil +} + +// CloseVolume disconnects sessions, removes from target, and closes the BlockVol. +func (m *VolumeManager) CloseVolume(name string) error { + m.mu.Lock() + defer m.mu.Unlock() + + mv, ok := m.volumes[name] + if !ok { + return nil // already closed, idempotent + } + + if m.target != nil { + m.target.DisconnectVolume(mv.iqn) + } + mv.vol.Close() + delete(m.volumes, name) + + m.logger.Printf("closed volume %q", name) + return nil +} + +// VolumeIQN returns the iSCSI IQN for a volume name. +func (m *VolumeManager) VolumeIQN(name string) string { + return m.volumeIQN(name) +} + +// VolumeExists returns true if the volume is currently tracked. +func (m *VolumeManager) VolumeExists(name string) bool { + m.mu.RLock() + defer m.mu.RUnlock() + _, ok := m.volumes[name] + return ok +} + +// VolumeSizeBytes returns the size of a tracked volume or 0 if not found. +func (m *VolumeManager) VolumeSizeBytes(name string) uint64 { + m.mu.RLock() + defer m.mu.RUnlock() + if mv, ok := m.volumes[name]; ok { + return mv.sizeBytes + } + return 0 +} + +// ListenAddr returns the target server's listen address. +func (m *VolumeManager) ListenAddr() string { + if m.target != nil { + return m.target.ListenAddr() + } + return "" +} + +func (m *VolumeManager) volumePath(name string) string { + return filepath.Join(m.dataDir, sanitizeFilename(name)+".blk") +} + +func (m *VolumeManager) volumeIQN(name string) string { + return m.iqnPrefix + ":" + SanitizeIQN(name) +} + +// sanitizeFilename delegates to the shared blockvol.SanitizeFilename. +func sanitizeFilename(name string) string { + return blockvol.SanitizeFilename(name) +} + +// SanitizeIQN delegates to the shared blockvol.SanitizeIQN. +func SanitizeIQN(name string) string { + return blockvol.SanitizeIQN(name) +} diff --git a/weed/storage/blockvol/csi/volume_manager_test.go b/weed/storage/blockvol/csi/volume_manager_test.go new file mode 100644 index 000000000..ef4102211 --- /dev/null +++ b/weed/storage/blockvol/csi/volume_manager_test.go @@ -0,0 +1,231 @@ +package csi + +import ( + "context" + "log" + "os" + "path/filepath" + "sync" + "testing" +) + +func newTestManager(t *testing.T) *VolumeManager { + t.Helper() + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-vm] ", log.LstdFlags) + mgr := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger) + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("start: %v", err) + } + t.Cleanup(func() { mgr.Stop() }) + return mgr +} + +func TestVolumeManager_CreateOpenClose(t *testing.T) { + mgr := newTestManager(t) + + if err := mgr.CreateVolume("vol1", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + if !mgr.VolumeExists("vol1") { + t.Fatal("expected vol1 to exist after create") + } + iqn := mgr.VolumeIQN("vol1") + if iqn == "" { + t.Fatal("expected non-empty IQN") + } + + // Close + if err := mgr.CloseVolume("vol1"); err != nil { + t.Fatalf("close: %v", err) + } + if mgr.VolumeExists("vol1") { + t.Fatal("expected vol1 to not exist after close") + } + + // Reopen + if err := mgr.OpenVolume("vol1"); err != nil { + t.Fatalf("reopen: %v", err) + } + if !mgr.VolumeExists("vol1") { + t.Fatal("expected vol1 to exist after reopen") + } +} + +func TestVolumeManager_DeleteRemovesFile(t *testing.T) { + mgr := newTestManager(t) + + if err := mgr.CreateVolume("delvol", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + + volPath := filepath.Join(mgr.dataDir, "delvol.blk") + if _, err := os.Stat(volPath); err != nil { + t.Fatalf("expected file to exist: %v", err) + } + + if err := mgr.DeleteVolume("delvol"); err != nil { + t.Fatalf("delete: %v", err) + } + + if _, err := os.Stat(volPath); !os.IsNotExist(err) { + t.Fatalf("expected file to be removed, got: %v", err) + } + if mgr.VolumeExists("delvol") { + t.Fatal("expected volume to not exist after delete") + } +} + +func TestVolumeManager_DuplicateCreate(t *testing.T) { + mgr := newTestManager(t) + + if err := mgr.CreateVolume("dup", 4*1024*1024); err != nil { + t.Fatalf("first create: %v", err) + } + // Same size -> idempotent success. + if err := mgr.CreateVolume("dup", 4*1024*1024); err != nil { + t.Fatalf("duplicate create (same size): expected success, got: %v", err) + } + // Larger size -> mismatch error. + err := mgr.CreateVolume("dup", 8*1024*1024) + if err != ErrVolumeSizeMismatch { + t.Fatalf("expected ErrVolumeSizeMismatch, got: %v", err) + } +} + +func TestVolumeManager_ListenAddr(t *testing.T) { + mgr := newTestManager(t) + + addr := mgr.ListenAddr() + if addr == "" { + t.Fatal("expected non-empty listen addr") + } +} + +func TestVolumeManager_OpenNonExistent(t *testing.T) { + mgr := newTestManager(t) + + err := mgr.OpenVolume("nonexistent") + if err == nil { + t.Fatal("expected error opening non-existent volume") + } +} + +func TestVolumeManager_CloseAlreadyClosed(t *testing.T) { + mgr := newTestManager(t) + + // Close a volume that was never opened -- should be idempotent. + if err := mgr.CloseVolume("nope"); err != nil { + t.Fatalf("close non-existent: %v", err) + } +} + +func TestVolumeManager_ConcurrentCreateDelete(t *testing.T) { + mgr := newTestManager(t) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + name := "conc" + string(rune('0'+i)) + wg.Add(1) + go func(n string) { + defer wg.Done() + if err := mgr.CreateVolume(n, 4*1024*1024); err != nil { + t.Errorf("create %s: %v", n, err) + return + } + if err := mgr.DeleteVolume(n); err != nil { + t.Errorf("delete %s: %v", n, err) + } + }(name) + } + wg.Wait() +} + +func TestVolumeManager_SanitizeIQN(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"pvc-abc123", "pvc-abc123"}, + {"PVC_ABC123", "pvc-abc123"}, + {"hello world!", "hello-world-"}, + {"a/b\\c:d", "a-b-c-d"}, + } + for _, tt := range tests { + got := SanitizeIQN(tt.input) + if got != tt.want { + t.Errorf("SanitizeIQN(%q): got %q, want %q", tt.input, got, tt.want) + } + } + + // Test truncation to 64 chars. + long := "" + for i := 0; i < 100; i++ { + long += "a" + } + if len(SanitizeIQN(long)) != 64 { + t.Fatalf("expected truncation to 64, got %d", len(SanitizeIQN(long))) + } +} + +// TestVolumeManager_CreateIdempotentAfterRestart simulates driver restart: +// existing .blk file on disk but not tracked in-memory. +func TestVolumeManager_CreateIdempotentAfterRestart(t *testing.T) { + dir := t.TempDir() + logger := log.New(os.Stderr, "[test-vm] ", log.LstdFlags) + + // Phase 1: create a volume, then stop the manager. + mgr1 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger) + if err := mgr1.Start(context.Background()); err != nil { + t.Fatalf("start1: %v", err) + } + if err := mgr1.CreateVolume("restart-vol", 4*1024*1024); err != nil { + t.Fatalf("create: %v", err) + } + mgr1.Stop() + + // Verify .blk file still exists on disk. + volPath := filepath.Join(dir, "restart-vol.blk") + if _, err := os.Stat(volPath); err != nil { + t.Fatalf("expected .blk file to exist: %v", err) + } + + // Phase 2: new manager (simulates restart) -- CreateVolume should + // adopt the existing file and return success. + mgr2 := NewVolumeManager(dir, "127.0.0.1:0", "iqn.2024.com.seaweedfs", logger) + if err := mgr2.Start(context.Background()); err != nil { + t.Fatalf("start2: %v", err) + } + defer mgr2.Stop() + + if err := mgr2.CreateVolume("restart-vol", 4*1024*1024); err != nil { + t.Fatalf("create after restart: expected idempotent success, got: %v", err) + } + if !mgr2.VolumeExists("restart-vol") { + t.Fatal("expected volume to be tracked after adoption") + } + if mgr2.VolumeSizeBytes("restart-vol") < 4*1024*1024 { + t.Fatalf("expected size >= 4MiB, got %d", mgr2.VolumeSizeBytes("restart-vol")) + } +} + +// TestVolumeManager_IQNCollision verifies that two long names sharing a prefix +// produce distinct IQNs after truncation. +func TestVolumeManager_IQNCollision(t *testing.T) { + prefix := "" + for i := 0; i < 70; i++ { + prefix += "a" + } + name1 := prefix + "-suffix1" + name2 := prefix + "-suffix2" + + iqn1 := SanitizeIQN(name1) + iqn2 := SanitizeIQN(name2) + + if iqn1 == iqn2 { + t.Fatalf("IQN collision: both names produced %q", iqn1) + } + if len(iqn1) > 64 || len(iqn2) > 64 { + t.Fatalf("IQN too long: %d, %d", len(iqn1), len(iqn2)) + } +} diff --git a/weed/storage/blockvol/iscsi/cmd/iscsi-target/main.go b/weed/storage/blockvol/iscsi/cmd/iscsi-target/main.go index 801299505..b121daa07 100644 --- a/weed/storage/blockvol/iscsi/cmd/iscsi-target/main.go +++ b/weed/storage/blockvol/iscsi/cmd/iscsi-target/main.go @@ -111,7 +111,7 @@ func main() { // Create Prometheus registry and metrics adapter. promReg := prometheus.NewRegistry() instrumented := &instrumentedAdapter{ - inner: &blockVolAdapter{vol: vol, tpgID: uint16(*tpgID)}, + inner: &blockvol.BlockVolAdapter{Vol: vol, TPGID: uint16(*tpgID)}, logger: logger, } adapter := newMetricsAdapter(instrumented, vol, promReg) @@ -173,61 +173,6 @@ func main() { logger.Println("target stopped") } -// blockVolAdapter wraps BlockVol to implement iscsi.BlockDevice and iscsi.ALUAProvider. -type blockVolAdapter struct { - vol *blockvol.BlockVol - tpgID uint16 -} - -func (a *blockVolAdapter) ReadAt(lba uint64, length uint32) ([]byte, error) { - return a.vol.ReadLBA(lba, length) -} -func (a *blockVolAdapter) WriteAt(lba uint64, data []byte) error { - return a.vol.WriteLBA(lba, data) -} -func (a *blockVolAdapter) Trim(lba uint64, length uint32) error { - return a.vol.Trim(lba, length) -} -func (a *blockVolAdapter) SyncCache() error { - return a.vol.SyncCache() -} -func (a *blockVolAdapter) BlockSize() uint32 { return a.vol.Info().BlockSize } -func (a *blockVolAdapter) VolumeSize() uint64 { return a.vol.Info().VolumeSize } -func (a *blockVolAdapter) IsHealthy() bool { return a.vol.Info().Healthy } - -// ALUAProvider implementation. -func (a *blockVolAdapter) ALUAState() uint8 { return roleToALUA(a.vol.Role()) } -func (a *blockVolAdapter) TPGroupID() uint16 { return a.tpgID } -func (a *blockVolAdapter) DeviceNAA() [8]byte { return uuidToNAA(a.vol.Info().UUID) } - -// roleToALUA maps a BlockVol Role to an ALUA asymmetric access state. -// RoleNone maps to Active/Optimized so standalone single-node targets -// (no assignment from master) can accept writes. -func roleToALUA(r blockvol.Role) uint8 { - switch r { - case blockvol.RolePrimary, blockvol.RoleNone: - return iscsi.ALUAActiveOptimized - case blockvol.RoleReplica: - return iscsi.ALUAStandby - case blockvol.RoleStale: - return iscsi.ALUAUnavailable - case blockvol.RoleRebuilding, blockvol.RoleDraining: - return iscsi.ALUATransitioning - default: - return iscsi.ALUAStandby - } -} - -// uuidToNAA converts a 16-byte UUID to an 8-byte NAA-6 identifier. -// NAA-6 format: nibble 6 (NAA=6) followed by 60 bits from the UUID. -func uuidToNAA(uuid [16]byte) [8]byte { - var naa [8]byte - // Set NAA=6 in the high nibble of the first byte. - naa[0] = 0x60 | (uuid[0] & 0x0F) - copy(naa[1:], uuid[1:8]) - return naa -} - // instrumentedAdapter wraps a BlockDevice and logs latency stats periodically. type instrumentedAdapter struct { inner iscsi.BlockDevice diff --git a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go index 00c41e858..0e48fd16d 100644 --- a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go +++ b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics.go @@ -9,7 +9,7 @@ import ( ) // metricsAdapter wraps a BlockDevice and feeds Prometheus counters/histograms. -// It sits in the adapter stack: metricsAdapter -> instrumentedAdapter -> blockVolAdapter. +// It sits in the adapter stack: metricsAdapter -> instrumentedAdapter -> BlockVolAdapter. // Counters count all attempts (including errors) per Prometheus conventions. type metricsAdapter struct { inner iscsi.BlockDevice diff --git a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics_test.go b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics_test.go index 4e0f624d5..24ac925ee 100644 --- a/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics_test.go +++ b/weed/storage/blockvol/iscsi/cmd/iscsi-target/metrics_test.go @@ -30,7 +30,7 @@ func TestMetrics_WriteIncrementsCounter(t *testing.T) { defer vol.Close() reg := prometheus.NewRegistry() - inner := &blockVolAdapter{vol: vol, tpgID: 1} + inner := &blockvol.BlockVolAdapter{Vol: vol, TPGID: 1} m := newMetricsAdapter(inner, vol, reg) // Write 10 blocks. @@ -80,7 +80,7 @@ func TestMetrics_EndpointServes(t *testing.T) { defer vol.Close() reg := prometheus.NewRegistry() - inner := &blockVolAdapter{vol: vol, tpgID: 1} + inner := &blockvol.BlockVolAdapter{Vol: vol, TPGID: 1} _ = newMetricsAdapter(inner, vol, reg) adm := newAdminServer(vol, "", log.New(os.Stderr, "[test] ", 0)) diff --git a/weed/storage/blockvol/iscsi/session.go b/weed/storage/blockvol/iscsi/session.go index fc12b6927..6fc0d78ee 100644 --- a/weed/storage/blockvol/iscsi/session.go +++ b/weed/storage/blockvol/iscsi/session.go @@ -34,12 +34,13 @@ const ( type Session struct { mu sync.Mutex - state SessionState - conn net.Conn - scsi *SCSIHandler - config TargetConfig - resolver TargetResolver - devices DeviceLookup + state SessionState + conn net.Conn + scsi *SCSIHandler + config TargetConfig + resolver TargetResolver + devices DeviceLookup + targetIQN string // negotiated target name (set after login) // Sequence numbers expCmdSN atomic.Uint32 // expected CmdSN from initiator @@ -315,6 +316,7 @@ func (s *Session) handleLogin(pdu *PDU) error { s.loginDone = true s.state = SessionLoggedIn result := s.negotiator.Result() + s.targetIQN = result.TargetName s.dataInWriter = NewDataInWriter(uint32(result.MaxRecvDataSegLen)) s.negImmediateData = result.ImmediateData s.negInitialR2T = result.InitialR2T @@ -593,3 +595,11 @@ func (s *Session) State() SessionState { defer s.mu.Unlock() return s.state } + +// TargetIQN returns the negotiated target name for this session. +// Returns empty string if login has not completed. +func (s *Session) TargetIQN() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.targetIQN +} diff --git a/weed/storage/blockvol/iscsi/target.go b/weed/storage/blockvol/iscsi/target.go index ea82a9d9a..db059b3bf 100644 --- a/weed/storage/blockvol/iscsi/target.go +++ b/weed/storage/blockvol/iscsi/target.go @@ -185,6 +185,20 @@ func (ts *TargetServer) LookupDevice(iqn string) BlockDevice { return &nullDevice{} } +// DisconnectVolume terminates all active sessions using the given IQN, +// then removes the volume from the registry. +func (ts *TargetServer) DisconnectVolume(iqn string) { + ts.activeMu.Lock() + for id, sess := range ts.active { + if sess.TargetIQN() == iqn { + sess.Close() + delete(ts.active, id) + } + } + ts.activeMu.Unlock() + ts.RemoveVolume(iqn) +} + // Close gracefully shuts down the target server. func (ts *TargetServer) Close() error { select { diff --git a/weed/storage/blockvol/naming.go b/weed/storage/blockvol/naming.go new file mode 100644 index 000000000..22d1d9041 --- /dev/null +++ b/weed/storage/blockvol/naming.go @@ -0,0 +1,31 @@ +package blockvol + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" +) + +var reInvalidFilename = regexp.MustCompile(`[^a-z0-9._-]`) +var reInvalidIQN = regexp.MustCompile(`[^a-z0-9.\-]`) + +// SanitizeFilename normalizes a volume name for use as a filename. +// Lowercases, replaces invalid chars with '-'. +func SanitizeFilename(name string) string { + return reInvalidFilename.ReplaceAllString(strings.ToLower(name), "-") +} + +// SanitizeIQN normalizes a CSI volume ID for use in an IQN. +// Lowercases, replaces invalid chars with '-', truncates to 64 chars. +// When truncation is needed, a hash suffix is appended to preserve uniqueness. +func SanitizeIQN(name string) string { + s := strings.ToLower(name) + s = reInvalidIQN.ReplaceAllString(s, "-") + if len(s) > 64 { + h := sha256.Sum256([]byte(name)) + suffix := hex.EncodeToString(h[:4]) // 8 hex chars + s = s[:64-1-len(suffix)] + "-" + suffix + } + return s +} diff --git a/weed/storage/blockvol/naming_test.go b/weed/storage/blockvol/naming_test.go new file mode 100644 index 000000000..2183baade --- /dev/null +++ b/weed/storage/blockvol/naming_test.go @@ -0,0 +1,79 @@ +package blockvol + +import ( + "strings" + "testing" +) + +func TestSanitizeFilename(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"simple", "simple"}, + {"VolA", "vola"}, + {"pvc-abc-123", "pvc-abc-123"}, + {"has spaces", "has-spaces"}, + {"UPPER_CASE", "upper_case"}, + {"special!@#$%chars", "special-----chars"}, + {"dots.and-dashes", "dots.and-dashes"}, + } + for _, tt := range tests { + got := SanitizeFilename(tt.input) + if got != tt.want { + t.Errorf("SanitizeFilename(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestSanitizeIQN(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"simple", "simple"}, + {"VolA", "vola"}, + {"pvc-abc-123", "pvc-abc-123"}, + {"has spaces", "has-spaces"}, + {"under_score", "under-score"}, + } + for _, tt := range tests { + got := SanitizeIQN(tt.input) + if got != tt.want { + t.Errorf("SanitizeIQN(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestSanitizeIQN_Truncation(t *testing.T) { + long := strings.Repeat("a", 100) + got := SanitizeIQN(long) + if len(got) > 64 { + t.Errorf("SanitizeIQN should truncate to 64 chars, got %d", len(got)) + } + // Should end with hash suffix. + if !strings.Contains(got, "-") { + t.Error("truncated IQN should have hash suffix separated by dash") + } +} + +func TestSanitizeConsistency(t *testing.T) { + // SanitizeFilename and SanitizeIQN should agree on lowercasing. + // "VolA" and "vola" should produce the same sanitized output from both. + names := []string{"VolA", "vola"} + for _, fn := range []struct { + name string + f func(string) string + }{ + {"SanitizeFilename", SanitizeFilename}, + {"SanitizeIQN", SanitizeIQN}, + } { + results := make(map[string]bool) + for _, n := range names { + results[fn.f(n)] = true + } + if len(results) != 1 { + t.Errorf("%s: 'VolA' and 'vola' should produce same result, got %v", fn.name, results) + } + } +} diff --git a/weed/storage/blockvol/test/apps_test.go b/weed/storage/blockvol/test/apps_test.go index 87eeb940c..2649e333a 100644 --- a/weed/storage/blockvol/test/apps_test.go +++ b/weed/storage/blockvol/test/apps_test.go @@ -303,9 +303,4 @@ func testAppsMdRaid1(t *testing.T) { t.Skip("MD RAID-1 test requires 2 iSCSI volumes") } -func requireCmd(t *testing.T, cmd string) { - t.Helper() - if !clientNode.HasCommand(cmd) { - t.Skipf("%s not available", cmd) - } -} +// requireCmd is defined in fault_helpers.go (integration tag). diff --git a/weed/storage/blockvol/test/fault_helpers.go b/weed/storage/blockvol/test/fault_helpers.go new file mode 100644 index 000000000..7c357e4db --- /dev/null +++ b/weed/storage/blockvol/test/fault_helpers.go @@ -0,0 +1,147 @@ +//go:build integration + +package test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +// requireCmd skips the test if cmd is not available on clientNode. +func requireCmd(t *testing.T, cmd string) { + t.Helper() + if !clientNode.HasCommand(cmd) { + t.Skipf("%s not available", cmd) + } +} + +// injectNetem adds a netem delay on the node's outbound traffic to targetIP. +// Returns a cleanup function that removes the qdisc. +// Requires tc (iproute2) and root access. +func injectNetem(ctx context.Context, node *Node, targetIP string, delayMs int) (cleanup func(), err error) { + // Find the interface routing to targetIP + iface, _, code, err := node.RunRoot(ctx, fmt.Sprintf( + "ip route get %s | head -1 | awk '{for(i=1;i<=NF;i++) if($i==\"dev\") print $(i+1)}'", targetIP)) + iface = strings.TrimSpace(iface) + if err != nil || code != 0 || iface == "" { + return nil, fmt.Errorf("find interface for %s: iface=%q code=%d err=%v", targetIP, iface, code, err) + } + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "tc qdisc add dev %s root netem delay %dms", iface, delayMs)) + if err != nil || code != 0 { + return nil, fmt.Errorf("tc qdisc add: code=%d stderr=%s err=%v", code, stderr, err) + } + + cleanup = func() { + cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + node.RunRoot(cctx, fmt.Sprintf("tc qdisc del dev %s root 2>/dev/null", iface)) + } + return cleanup, nil +} + +// injectIptablesDrop blocks outbound TCP traffic from node to targetIP on the given ports. +// Returns a cleanup function that removes the iptables rules. +func injectIptablesDrop(ctx context.Context, node *Node, targetIP string, ports []int) (cleanup func(), err error) { + for _, port := range ports { + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "iptables -A OUTPUT -d %s -p tcp --dport %d -j DROP", targetIP, port)) + if err != nil || code != 0 { + // Rollback already-added rules + for _, p2 := range ports { + if p2 == port { + break + } + node.RunRoot(ctx, fmt.Sprintf( + "iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, p2)) + } + return nil, fmt.Errorf("iptables add port %d: code=%d stderr=%s err=%v", port, code, stderr, err) + } + } + + cleanup = func() { + cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for _, port := range ports { + node.RunRoot(cctx, fmt.Sprintf( + "iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, port)) + } + } + return cleanup, nil +} + +// fillDisk fills the filesystem at dir, leaving ~4MB free. +// Returns a cleanup function that removes the fill file. +func fillDisk(ctx context.Context, node *Node, dir string) (cleanup func(), err error) { + // Get available space in MB + stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf( + "df -BM --output=avail %s | tail -1 | tr -d ' M'", dir)) + if err != nil || code != 0 { + return nil, fmt.Errorf("df: code=%d err=%v", code, err) + } + availMB := 0 + fmt.Sscanf(strings.TrimSpace(stdout), "%d", &availMB) + if availMB < 8 { + return nil, fmt.Errorf("not enough space to fill: %dMB available", availMB) + } + fillMB := availMB - 4 // leave 4MB + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "dd if=/dev/zero of=%s/fillfile bs=1M count=%d 2>/dev/null", dir, fillMB)) + if err != nil || code != 0 { + // dd may return non-zero on ENOSPC which is expected; check if file was created + stdout2, _, _, _ := node.RunRoot(ctx, fmt.Sprintf("test -f %s/fillfile && echo ok", dir)) + if !strings.Contains(stdout2, "ok") { + return nil, fmt.Errorf("fillDisk dd: code=%d stderr=%s err=%v", code, stderr, err) + } + } + + cleanup = func() { + cctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + node.RunRoot(cctx, fmt.Sprintf("rm -f %s/fillfile", dir)) + } + return cleanup, nil +} + +// corruptWALRegion overwrites nBytes within the WAL section of the volume file. +// The WAL is embedded in the volume file starting at offset 4096 (SuperblockSize). +// We corrupt near the end of the WAL region to simulate torn writes. +func corruptWALRegion(ctx context.Context, node *Node, volPath string, nBytes int) error { + const walOffset = 4096 // SuperblockSize — WAL starts here + + // Get file size to determine WAL region extent + stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf("stat -c %%s %s", volPath)) + if err != nil || code != 0 { + return fmt.Errorf("stat %s: code=%d err=%v", volPath, code, err) + } + fileSize := 0 + fmt.Sscanf(strings.TrimSpace(stdout), "%d", &fileSize) + + // WAL region is from walOffset to walOffset + walSize. + // For a 50M vol with default 64M WAL, WAL extends from 4096 to ~67M. + // Corrupt nBytes at a position 1/3 into the WAL region (where recent writes live). + walEnd := walOffset + 64*1024*1024 // default 64MB WAL + if walEnd > fileSize { + walEnd = fileSize + } + walUsable := walEnd - walOffset + if walUsable < nBytes*2 { + return fmt.Errorf("WAL region too small: %d", walUsable) + } + // Corrupt near the middle of the WAL region + seekPos := walOffset + walUsable/3 + + _, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf( + "python3 -c \"import sys; sys.stdout.buffer.write(b'\\xff'*%d)\" | dd of=%s bs=1 seek=%d conv=notrunc 2>/dev/null", + nBytes, volPath, seekPos)) + if err != nil || code != 0 { + return fmt.Errorf("corrupt WAL region: code=%d stderr=%s err=%v", code, stderr, err) + } + return nil +} +