diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.toml new file mode 100644 index 000000000..fa9272b9c --- /dev/null +++ b/seaweed-volume/Cargo.toml @@ -0,0 +1,81 @@ +[package] +name = "seaweed-volume" +version = "0.1.0" +edition = "2021" +description = "SeaweedFS Volume Server — Rust implementation" + +[dependencies] +# Async runtime +tokio = { version = "1", features = ["full"] } + +# gRPC + protobuf +tonic = { version = "0.12", features = ["tls"] } +prost = "0.13" +prost-types = "0.13" + +# HTTP server +axum = { version = "0.7", features = ["multipart"] } +hyper = { version = "1", features = ["full"] } +hyper-util = { version = "0.1", features = ["tokio"] } +tower = "0.4" +tower-http = { version = "0.5", features = ["cors", "trace"] } + +# CLI +clap = { version = "4", features = ["derive"] } + +# Metrics +prometheus = { version = "0.13", features = ["process"] } + +# JWT +jsonwebtoken = "9" + +# TLS +rustls = "0.23" +tokio-rustls = "0.26" +rustls-pemfile = "2" + +# LevelDB (via RocksDB for better Rust support) +# Using rusty-leveldb for pure Rust LevelDB +rusty-leveldb = "3" + +# Reed-Solomon erasure coding +reed-solomon-erasure = "6" + +# Logging +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +# Config +toml = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +# CRC32 +crc32fast = "1" + +# Memory-mapped files +memmap2 = "0.9" + +# UUID +uuid = { version = "1", features = ["v4"] } + +# HTTP client (for proxying, remote fetch) +reqwest = { version = "0.12", features = ["rustls-tls", "stream"] } + +# Misc +bytes = "1" +rand = "0.8" +chrono = "0.4" +hex = "0.4" +parking_lot = "0.12" +dashmap = "6" +thiserror = "1" +anyhow = "1" +async-trait = "0.1" +futures = "0.3" + +# Disk space checking +sysinfo = "0.31" + +[build-dependencies] +tonic-build = "0.12" diff --git a/seaweed-volume/build.rs b/seaweed-volume/build.rs new file mode 100644 index 000000000..1ef8b0699 --- /dev/null +++ b/seaweed-volume/build.rs @@ -0,0 +1,14 @@ +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(true) + .build_client(true) + .compile_protos( + &[ + "proto/volume_server.proto", + "proto/master.proto", + "proto/remote.proto", + ], + &["proto/"], + )?; + Ok(()) +} diff --git a/seaweed-volume/proto/master.proto b/seaweed-volume/proto/master.proto new file mode 100644 index 000000000..8289cd233 --- /dev/null +++ b/seaweed-volume/proto/master.proto @@ -0,0 +1,474 @@ +syntax = "proto3"; + +package master_pb; + +option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"; + +import "volume_server.proto"; + +////////////////////////////////////////////////// + +service Seaweed { + rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { + } + rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { + } + rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { + } + rpc Assign (AssignRequest) returns (AssignResponse) { + } + rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) { + } + rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { + } + rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { + } + rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) { + } + rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) { + } + rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) { + } + rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) { + } + rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) { + } + rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) { + } + rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { + } + rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) { + } + rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) { + } + rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) { + } + rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) { + } + rpc Ping (PingRequest) returns (PingResponse) { + } + rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) { + } + rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) { + } + rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) { + } + rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) { + } + rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { + } +} + +////////////////////////////////////////////////// + +message DiskTag { + uint32 disk_id = 1; + repeated string tags = 2; +} + +message Heartbeat { + string ip = 1; + uint32 port = 2; + string public_url = 3; + uint64 max_file_key = 5; + string data_center = 6; + string rack = 7; + uint32 admin_port = 8; + repeated VolumeInformationMessage volumes = 9; + // delta volumes + repeated VolumeShortInformationMessage new_volumes = 10; + repeated VolumeShortInformationMessage deleted_volumes = 11; + bool has_no_volumes = 12; + + // erasure coding + repeated VolumeEcShardInformationMessage ec_shards = 16; + // delta erasure coding shards + repeated VolumeEcShardInformationMessage new_ec_shards = 17; + repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; + bool has_no_ec_shards = 19; + + map max_volume_counts = 4; + uint32 grpc_port = 20; + repeated string location_uuids = 21; + string id = 22; // volume server id, independent of ip:port for stable identification + + // state flags + volume_server_pb.VolumeServerState state = 23; + + repeated DiskTag disk_tags = 24; +} + +message HeartbeatResponse { + uint64 volume_size_limit = 1; + string leader = 2; + string metrics_address = 3; + uint32 metrics_interval_seconds = 4; + repeated StorageBackend storage_backends = 5; + repeated string duplicated_uuids = 6; + bool preallocate = 7; +} + +message VolumeInformationMessage { + uint32 id = 1; + uint64 size = 2; + string collection = 3; + uint64 file_count = 4; + uint64 delete_count = 5; + uint64 deleted_byte_count = 6; + bool read_only = 7; + uint32 replica_placement = 8; + uint32 version = 9; + uint32 ttl = 10; + uint32 compact_revision = 11; + int64 modified_at_second = 12; + string remote_storage_name = 13; + string remote_storage_key = 14; + string disk_type = 15; + uint32 disk_id = 16; +} + +message VolumeShortInformationMessage { + uint32 id = 1; + string collection = 3; + uint32 replica_placement = 8; + uint32 version = 9; + uint32 ttl = 10; + string disk_type = 15; + uint32 disk_id = 16; +} + +message VolumeEcShardInformationMessage { + uint32 id = 1; + string collection = 2; + uint32 ec_index_bits = 3; + string disk_type = 4; + uint64 expire_at_sec = 5; // used to record the destruction time of ec volume + uint32 disk_id = 6; + repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits +} + +message StorageBackend { + string type = 1; + string id = 2; + map properties = 3; +} + +message Empty { +} + +message SuperBlockExtra { + message ErasureCoding { + uint32 data = 1; + uint32 parity = 2; + repeated uint32 volume_ids = 3; + } + ErasureCoding erasure_coding = 1; +} + +message KeepConnectedRequest { + string client_type = 1; + string client_address = 3; + string version = 4; + string filer_group = 5; + string data_center = 6; + string rack = 7; +} + +message VolumeLocation { + string url = 1; + string public_url = 2; + repeated uint32 new_vids = 3; + repeated uint32 deleted_vids = 4; + string leader = 5; // optional when leader is not itself + string data_center = 6; // optional when DataCenter is in use + uint32 grpc_port = 7; + repeated uint32 new_ec_vids = 8; + repeated uint32 deleted_ec_vids = 9; +} + +message ClusterNodeUpdate { + string node_type = 1; + string address = 2; + bool is_add = 4; + string filer_group = 5; + int64 created_at_ns = 6; +} + +message KeepConnectedResponse { + VolumeLocation volume_location = 1; + ClusterNodeUpdate cluster_node_update = 2; +} + +message LookupVolumeRequest { + repeated string volume_or_file_ids = 1; + string collection = 2; // optional, a bit faster if provided. +} +message LookupVolumeResponse { + message VolumeIdLocation { + string volume_or_file_id = 1; + repeated Location locations = 2; + string error = 3; + string auth = 4; + } + repeated VolumeIdLocation volume_id_locations = 1; +} + +message Location { + string url = 1; + string public_url = 2; + uint32 grpc_port = 3; + string data_center = 4; +} + +message AssignRequest { + uint64 count = 1; + string replication = 2; + string collection = 3; + string ttl = 4; + string data_center = 5; + string rack = 6; + string data_node = 7; + uint32 memory_map_max_size_mb = 8; + uint32 writable_volume_count = 9; + string disk_type = 10; +} + +message VolumeGrowRequest { + uint32 writable_volume_count = 1; + string replication = 2; + string collection = 3; + string ttl = 4; + string data_center = 5; + string rack = 6; + string data_node = 7; + uint32 memory_map_max_size_mb = 8; + string disk_type = 9; +} + +message AssignResponse { + string fid = 1; + uint64 count = 4; + string error = 5; + string auth = 6; + repeated Location replicas = 7; + Location location = 8; +} + +message StatisticsRequest { + string replication = 1; + string collection = 2; + string ttl = 3; + string disk_type = 4; +} +message StatisticsResponse { + uint64 total_size = 4; + uint64 used_size = 5; + uint64 file_count = 6; +} + +// +// collection related +// +message Collection { + string name = 1; +} +message CollectionListRequest { + bool include_normal_volumes = 1; + bool include_ec_volumes = 2; +} +message CollectionListResponse { + repeated Collection collections = 1; +} + +message CollectionDeleteRequest { + string name = 1; +} +message CollectionDeleteResponse { +} + +// +// volume related +// +message DiskInfo { + string type = 1; + int64 volume_count = 2; + int64 max_volume_count = 3; + int64 free_volume_count = 4; + int64 active_volume_count = 5; + repeated VolumeInformationMessage volume_infos = 6; + repeated VolumeEcShardInformationMessage ec_shard_infos = 7; + int64 remote_volume_count = 8; + uint32 disk_id = 9; + repeated string tags = 10; +} +message DataNodeInfo { + string id = 1; + map diskInfos = 2; + uint32 grpc_port = 3; + string address = 4; // ip:port for connecting to the volume server +} +message RackInfo { + string id = 1; + repeated DataNodeInfo data_node_infos = 2; + map diskInfos = 3; +} +message DataCenterInfo { + string id = 1; + repeated RackInfo rack_infos = 2; + map diskInfos = 3; +} +message TopologyInfo { + string id = 1; + repeated DataCenterInfo data_center_infos = 2; + map diskInfos = 3; +} +message VolumeListRequest { +} +message VolumeListResponse { + TopologyInfo topology_info = 1; + uint64 volume_size_limit_mb = 2; +} + +message LookupEcVolumeRequest { + uint32 volume_id = 1; +} +message LookupEcVolumeResponse { + uint32 volume_id = 1; + message EcShardIdLocation { + uint32 shard_id = 1; + repeated Location locations = 2; + } + repeated EcShardIdLocation shard_id_locations = 2; +} + +message VacuumVolumeRequest { + float garbage_threshold = 1; + uint32 volume_id = 2; + string collection = 3; +} +message VacuumVolumeResponse { +} + +message DisableVacuumRequest { +} +message DisableVacuumResponse { +} + +message EnableVacuumRequest { +} +message EnableVacuumResponse { +} + +message VolumeMarkReadonlyRequest { + string ip = 1; + uint32 port = 2; + uint32 volume_id = 4; + string collection = 5; + uint32 replica_placement = 6; + uint32 version = 7; + uint32 ttl = 8; + string disk_type = 9; + bool is_readonly = 10; +} +message VolumeMarkReadonlyResponse { +} + +message GetMasterConfigurationRequest { +} +message GetMasterConfigurationResponse { + string metrics_address = 1; + uint32 metrics_interval_seconds = 2; + repeated StorageBackend storage_backends = 3; + string default_replication = 4; + string leader = 5; + uint32 volume_size_limit_m_b = 6; + bool volume_preallocate = 7; + // MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027. + string maintenance_scripts = 8; + uint32 maintenance_sleep_minutes = 9; +} + +message ListClusterNodesRequest { + string client_type = 1; + string filer_group = 2; + int32 limit = 4; +} +message ListClusterNodesResponse { + message ClusterNode { + string address = 1; + string version = 2; + int64 created_at_ns = 4; + string data_center = 5; + string rack = 6; + } + repeated ClusterNode cluster_nodes = 1; +} + +message LeaseAdminTokenRequest { + int64 previous_token = 1; + int64 previous_lock_time = 2; + string lock_name = 3; + string client_name = 4; + string message = 5; +} +message LeaseAdminTokenResponse { + int64 token = 1; + int64 lock_ts_ns = 2; +} + +message ReleaseAdminTokenRequest { + int64 previous_token = 1; + int64 previous_lock_time = 2; + string lock_name = 3; +} +message ReleaseAdminTokenResponse { +} + +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +} + +message RaftAddServerRequest { + string id = 1; + string address = 2; + bool voter = 3; +} +message RaftAddServerResponse { +} + +message RaftRemoveServerRequest { + string id = 1; + bool force = 2; +} +message RaftRemoveServerResponse { +} + +message RaftListClusterServersRequest { +} +message RaftListClusterServersResponse { + message ClusterServers { + string id = 1; + string address = 2; + string suffrage = 3; + bool isLeader = 4; + } + repeated ClusterServers cluster_servers = 1; +} + +message RaftLeadershipTransferRequest { + string target_id = 1; // Optional: target server ID. If empty, transfers to any eligible follower + string target_address = 2; // Optional: target server address. Required if target_id is specified +} +message RaftLeadershipTransferResponse { + string previous_leader = 1; + string new_leader = 2; +} + +message VolumeGrowResponse { +} diff --git a/seaweed-volume/proto/remote.proto b/seaweed-volume/proto/remote.proto new file mode 100644 index 000000000..9d6d81ff5 --- /dev/null +++ b/seaweed-volume/proto/remote.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; + +package remote_pb; + +option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"; +option java_package = "seaweedfs.client"; +option java_outer_classname = "FilerProto"; + +///////////////////////// +// Remote Storage related +///////////////////////// +message RemoteConf { + string type = 1; + string name = 2; + string s3_access_key = 4; + string s3_secret_key = 5; + string s3_region = 6; + string s3_endpoint = 7; + string s3_storage_class = 8; + bool s3_force_path_style = 9; + bool s3_support_tagging = 13; + bool s3_v4_signature = 11; + + string gcs_google_application_credentials = 10; + string gcs_project_id = 12; + + string azure_account_name = 15; + string azure_account_key = 16; + + string backblaze_key_id = 20; + string backblaze_application_key = 21; + string backblaze_endpoint = 22; + string backblaze_region = 23; + + string aliyun_access_key = 25; + string aliyun_secret_key = 26; + string aliyun_endpoint = 27; + string aliyun_region = 28; + + string tencent_secret_id = 30; + string tencent_secret_key = 31; + string tencent_endpoint = 32; + + string baidu_access_key = 35; + string baidu_secret_key = 36; + string baidu_endpoint = 37; + string baidu_region = 38; + + string wasabi_access_key = 40; + string wasabi_secret_key = 41; + string wasabi_endpoint = 42; + string wasabi_region = 43; + + string filebase_access_key = 60; + string filebase_secret_key = 61; + string filebase_endpoint = 62; + + string storj_access_key = 65; + string storj_secret_key = 66; + string storj_endpoint = 67; + + string contabo_access_key = 68; + string contabo_secret_key = 69; + string contabo_endpoint = 70; + string contabo_region = 71; +} + +message RemoteStorageMapping { + map mappings = 1; + string primary_bucket_storage_name = 2; +} +message RemoteStorageLocation { + string name = 1; + string bucket = 2; + string path = 3; +} diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto new file mode 100644 index 000000000..c1a9282bd --- /dev/null +++ b/seaweed-volume/proto/volume_server.proto @@ -0,0 +1,758 @@ +syntax = "proto3"; + +package volume_server_pb; +option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"; + +import "remote.proto"; + +////////////////////////////////////////////////// + +// Persistent state for volume servers. +message VolumeServerState { + // whether the server is in maintenance (i.e. read-only) mode. + bool maintenance = 1; + // incremental version counter + uint32 version = 2; +} + +////////////////////////////////////////////////// + +service VolumeServer { + //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. + rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) { + } + + rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) { + } + rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) { + } + rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) { + } + rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) { + } + + rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) { + } + rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) { + } + + rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) { + } + rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) { + } + + rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) { + } + rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) { + } + rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) { + } + rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { + } + rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) { + } + rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) { + } + rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) { + } + + rpc GetState (GetStateRequest) returns (GetStateResponse) { + } + rpc SetState (SetStateRequest) returns (SetStateResponse) { + } + + // copy the .idx .dat files, and mount this volume + rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) { + } + rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) { + } + rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) { + } + rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) { + } + + rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) { + } + rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) { + } + rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) { + } + rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) { + } + + rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) { + } + rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) { + } + + // erasure coding + rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) { + } + rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) { + } + rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) { + } + rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) { + } + rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) { + } + rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) { + } + rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) { + } + rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) { + } + rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) { + } + rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) { + } + + // tiered storage + rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) { + } + rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) { + } + + rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) { + } + rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) { + } + + // remote storage + rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) { + } + + // scrubbing + rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) { + } + rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) { + } + + // query + rpc Query (QueryRequest) returns (stream QueriedStripe) { + } + + rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) { + } + + rpc Ping (PingRequest) returns (PingResponse) { + } + +} + +////////////////////////////////////////////////// + +message BatchDeleteRequest { + repeated string file_ids = 1; + bool skip_cookie_check = 2; +} + +message BatchDeleteResponse { + repeated DeleteResult results = 1; +} +message DeleteResult { + string file_id = 1; + int32 status = 2; + string error = 3; + uint32 size = 4; + uint32 version = 5; +} + +message Empty { +} + +message VacuumVolumeCheckRequest { + uint32 volume_id = 1; +} +message VacuumVolumeCheckResponse { + double garbage_ratio = 1; +} + +message VacuumVolumeCompactRequest { + uint32 volume_id = 1; + int64 preallocate = 2; +} +message VacuumVolumeCompactResponse { + int64 processed_bytes = 1; + float load_avg_1m = 2; +} + +message VacuumVolumeCommitRequest { + uint32 volume_id = 1; +} +message VacuumVolumeCommitResponse { + bool is_read_only = 1; + uint64 volume_size = 2; +} + +message VacuumVolumeCleanupRequest { + uint32 volume_id = 1; +} +message VacuumVolumeCleanupResponse { +} + +message DeleteCollectionRequest { + string collection = 1; +} +message DeleteCollectionResponse { +} + +message AllocateVolumeRequest { + uint32 volume_id = 1; + string collection = 2; + int64 preallocate = 3; + string replication = 4; + string ttl = 5; + uint32 memory_map_max_size_mb = 6; + string disk_type = 7; + uint32 version = 8; +} +message AllocateVolumeResponse { +} + +message VolumeSyncStatusRequest { + uint32 volume_id = 1; +} +message VolumeSyncStatusResponse { + uint32 volume_id = 1; + string collection = 2; + string replication = 4; + string ttl = 5; + uint64 tail_offset = 6; + uint32 compact_revision = 7; + uint64 idx_file_size = 8; + uint32 version = 9; +} + +message VolumeIncrementalCopyRequest { + uint32 volume_id = 1; + uint64 since_ns = 2; +} +message VolumeIncrementalCopyResponse { + bytes file_content = 1; +} + +message VolumeMountRequest { + uint32 volume_id = 1; +} +message VolumeMountResponse { +} + +message VolumeUnmountRequest { + uint32 volume_id = 1; +} +message VolumeUnmountResponse { +} + +message VolumeDeleteRequest { + uint32 volume_id = 1; + bool only_empty = 2; +} +message VolumeDeleteResponse { +} + +message VolumeMarkReadonlyRequest { + uint32 volume_id = 1; + bool persist = 2; +} +message VolumeMarkReadonlyResponse { +} + +message VolumeMarkWritableRequest { + uint32 volume_id = 1; +} +message VolumeMarkWritableResponse { +} + +message VolumeConfigureRequest { + uint32 volume_id = 1; + string replication = 2; +} +message VolumeConfigureResponse { + string error = 1; +} + +message VolumeStatusRequest { + uint32 volume_id = 1; +} +message VolumeStatusResponse { + bool is_read_only = 1; + uint64 volume_size = 2; + uint64 file_count = 3; + uint64 file_deleted_count = 4; +} + +message GetStateRequest { +} +message GetStateResponse { + VolumeServerState state = 1; +} + +message SetStateRequest { + // SetState updates *all* volume server flags at once. Retrieve state with GetState(), + // modify individual flags as required, then call this RPC to update. + VolumeServerState state = 1; +} +message SetStateResponse { + VolumeServerState state = 1; +} + +message VolumeCopyRequest { + uint32 volume_id = 1; + string collection = 2; + string replication = 3; + string ttl = 4; + string source_data_node = 5; + string disk_type = 6; + int64 io_byte_per_second = 7; +} +message VolumeCopyResponse { + uint64 last_append_at_ns = 1; + int64 processed_bytes = 2; +} + +message CopyFileRequest { + uint32 volume_id = 1; + string ext = 2; + uint32 compaction_revision = 3; + uint64 stop_offset = 4; + string collection = 5; + bool is_ec_volume = 6; + bool ignore_source_file_not_found = 7; +} +message CopyFileResponse { + bytes file_content = 1; + int64 modified_ts_ns = 2; +} + +message ReceiveFileRequest { + oneof data { + ReceiveFileInfo info = 1; + bytes file_content = 2; + } +} + +message ReceiveFileInfo { + uint32 volume_id = 1; + string ext = 2; + string collection = 3; + bool is_ec_volume = 4; + uint32 shard_id = 5; + uint64 file_size = 6; +} + +message ReceiveFileResponse { + uint64 bytes_written = 1; + string error = 2; +} + +message ReadNeedleBlobRequest { + uint32 volume_id = 1; + int64 offset = 3; // actual offset + int32 size = 4; +} +message ReadNeedleBlobResponse { + bytes needle_blob = 1; +} + +message ReadNeedleMetaRequest { + uint32 volume_id = 1; + uint64 needle_id = 2; + int64 offset = 3; // actual offset + int32 size = 4; +} +message ReadNeedleMetaResponse { + uint32 cookie = 1; + uint64 last_modified = 2; + uint32 crc = 3; + string ttl = 4; + uint64 append_at_ns = 5; +} + +message WriteNeedleBlobRequest { + uint32 volume_id = 1; + uint64 needle_id = 2; + int32 size = 3; + bytes needle_blob = 4; +} +message WriteNeedleBlobResponse { +} + +message ReadAllNeedlesRequest { + repeated uint32 volume_ids = 1; +} +message ReadAllNeedlesResponse { + uint32 volume_id = 1; + uint64 needle_id = 2; + uint32 cookie = 3; + bytes needle_blob = 5; + bool needle_blob_compressed = 6; + uint64 last_modified = 7; + uint32 crc = 8; + bytes name = 9; + bytes mime = 10; +} + +message VolumeTailSenderRequest { + uint32 volume_id = 1; + uint64 since_ns = 2; + uint32 idle_timeout_seconds = 3; +} +message VolumeTailSenderResponse { + bytes needle_header = 1; + bytes needle_body = 2; + bool is_last_chunk = 3; + uint32 version = 4; +} + +message VolumeTailReceiverRequest { + uint32 volume_id = 1; + uint64 since_ns = 2; + uint32 idle_timeout_seconds = 3; + string source_volume_server = 4; +} +message VolumeTailReceiverResponse { +} + +message VolumeEcShardsGenerateRequest { + uint32 volume_id = 1; + string collection = 2; +} +message VolumeEcShardsGenerateResponse { +} + +message VolumeEcShardsRebuildRequest { + uint32 volume_id = 1; + string collection = 2; +} +message VolumeEcShardsRebuildResponse { + repeated uint32 rebuilt_shard_ids = 1; +} + +message VolumeEcShardsCopyRequest { + uint32 volume_id = 1; + string collection = 2; + repeated uint32 shard_ids = 3; + bool copy_ecx_file = 4; + string source_data_node = 5; + bool copy_ecj_file = 6; + bool copy_vif_file = 7; + uint32 disk_id = 8; // Target disk ID for storing EC shards +} +message VolumeEcShardsCopyResponse { +} + +message VolumeEcShardsDeleteRequest { + uint32 volume_id = 1; + string collection = 2; + repeated uint32 shard_ids = 3; +} +message VolumeEcShardsDeleteResponse { +} + +message VolumeEcShardsMountRequest { + uint32 volume_id = 1; + string collection = 2; + repeated uint32 shard_ids = 3; +} +message VolumeEcShardsMountResponse { +} + +message VolumeEcShardsUnmountRequest { + uint32 volume_id = 1; + repeated uint32 shard_ids = 3; +} +message VolumeEcShardsUnmountResponse { +} + +message VolumeEcShardReadRequest { + uint32 volume_id = 1; + uint32 shard_id = 2; + int64 offset = 3; + int64 size = 4; + uint64 file_key = 5; +} +message VolumeEcShardReadResponse { + bytes data = 1; + bool is_deleted = 2; +} + +message VolumeEcBlobDeleteRequest { + uint32 volume_id = 1; + string collection = 2; + uint64 file_key = 3; + uint32 version = 4; +} +message VolumeEcBlobDeleteResponse { +} + +message VolumeEcShardsToVolumeRequest { + uint32 volume_id = 1; + string collection = 2; +} +message VolumeEcShardsToVolumeResponse { +} + +message VolumeEcShardsInfoRequest { + uint32 volume_id = 1; +} +message VolumeEcShardsInfoResponse { + repeated EcShardInfo ec_shard_infos = 1; + uint64 volume_size = 2; + uint64 file_count = 3; + uint64 file_deleted_count = 4; +} + +message EcShardInfo { + uint32 shard_id = 1; + int64 size = 2; + string collection = 3; + uint32 volume_id = 4; +} + +message ReadVolumeFileStatusRequest { + uint32 volume_id = 1; +} +message ReadVolumeFileStatusResponse { + uint32 volume_id = 1; + uint64 idx_file_timestamp_seconds = 2; + uint64 idx_file_size = 3; + uint64 dat_file_timestamp_seconds = 4; + uint64 dat_file_size = 5; + uint64 file_count = 6; + uint32 compaction_revision = 7; + string collection = 8; + string disk_type = 9; + VolumeInfo volume_info = 10; + uint32 version = 11; +} + +message DiskStatus { + string dir = 1; + uint64 all = 2; + uint64 used = 3; + uint64 free = 4; + float percent_free = 5; + float percent_used = 6; + string disk_type = 7; +} + +message MemStatus { + int32 goroutines = 1; + uint64 all = 2; + uint64 used = 3; + uint64 free = 4; + uint64 self = 5; + uint64 heap = 6; + uint64 stack = 7; +} + +// tired storage on volume servers +message RemoteFile { + string backend_type = 1; + string backend_id = 2; + string key = 3; + uint64 offset = 4; + uint64 file_size = 5; + uint64 modified_time = 6; + string extension = 7; +} +message VolumeInfo { + repeated RemoteFile files = 1; + uint32 version = 2; + string replication = 3; + uint32 bytes_offset = 4; + int64 dat_file_size = 5; // store the original dat file size + uint64 expire_at_sec = 6; // expiration time of ec volume + bool read_only = 7; + EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4) +} + +// EcShardConfig specifies erasure coding shard configuration +message EcShardConfig { + uint32 data_shards = 1; // Number of data shards (e.g., 10) + uint32 parity_shards = 2; // Number of parity shards (e.g., 4) +} +message OldVersionVolumeInfo { + repeated RemoteFile files = 1; + uint32 version = 2; + string replication = 3; + uint32 BytesOffset = 4; + int64 dat_file_size = 5; // store the original dat file size + uint64 DestroyTime = 6; // expiration time of ec volume + bool read_only = 7; +} + +// tiered storage +message VolumeTierMoveDatToRemoteRequest { + uint32 volume_id = 1; + string collection = 2; + string destination_backend_name = 3; + bool keep_local_dat_file = 4; +} +message VolumeTierMoveDatToRemoteResponse { + int64 processed = 1; + float processedPercentage = 2; +} + +message VolumeTierMoveDatFromRemoteRequest { + uint32 volume_id = 1; + string collection = 2; + bool keep_remote_dat_file = 3; +} +message VolumeTierMoveDatFromRemoteResponse { + int64 processed = 1; + float processedPercentage = 2; +} + +message VolumeServerStatusRequest { + +} +message VolumeServerStatusResponse { + repeated DiskStatus disk_statuses = 1; + MemStatus memory_status = 2; + string version = 3; + string data_center = 4; + string rack = 5; + VolumeServerState state = 6; +} + +message VolumeServerLeaveRequest { +} +message VolumeServerLeaveResponse { +} + +// remote storage +message FetchAndWriteNeedleRequest { + uint32 volume_id = 1; + uint64 needle_id = 2; + uint32 cookie = 3; + int64 offset = 4; + int64 size = 5; + message Replica { + string url = 1; + string public_url = 2; + int32 grpc_port = 3; + } + repeated Replica replicas = 6; + string auth = 7; + // remote conf + remote_pb.RemoteConf remote_conf = 15; + remote_pb.RemoteStorageLocation remote_location = 16; +} +message FetchAndWriteNeedleResponse { + string e_tag = 1; +} + +enum VolumeScrubMode { + UNKNOWN = 0; + INDEX = 1; + FULL = 2; + LOCAL = 3; +} + +message ScrubVolumeRequest { + VolumeScrubMode mode = 1; + // optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed. + repeated uint32 volume_ids = 2; +} +message ScrubVolumeResponse { + uint64 total_volumes = 1; + uint64 total_files = 2; + repeated uint32 broken_volume_ids = 3; + repeated string details = 4; +} + +message ScrubEcVolumeRequest { + VolumeScrubMode mode = 1; + // optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed. + repeated uint32 volume_ids = 2; +} +message ScrubEcVolumeResponse { + uint64 total_volumes = 1; + uint64 total_files = 2; + repeated uint32 broken_volume_ids = 3; + repeated EcShardInfo broken_shard_infos = 4; + repeated string details = 5; +} + +// select on volume servers +message QueryRequest { + repeated string selections = 1; + repeated string from_file_ids = 2; + message Filter { + string field = 1; + string operand = 2; + string value = 3; + } + Filter filter = 3; + + message InputSerialization { + // NONE | GZIP | BZIP2 + string compression_type = 1; + message CSVInput { + string file_header_info = 1; // Valid values: NONE | USE | IGNORE + string record_delimiter = 2; // Default: \n + string field_delimiter = 3; // Default: , + string quote_character = 4; // Default: " + string quote_escape_character = 5; // Default: " + string comments = 6; // Default: # + // If true, records might contain record delimiters within quote characters + bool allow_quoted_record_delimiter = 7; // default False. + } + message JSONInput { + string type = 1; // Valid values: DOCUMENT | LINES + } + message ParquetInput { + } + + CSVInput csv_input = 2; + JSONInput json_input = 3; + ParquetInput parquet_input = 4; + } + InputSerialization input_serialization = 4; + + message OutputSerialization { + message CSVOutput { + string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED + string record_delimiter = 2; // Default: \n + string field_delimiter = 3; // Default: , + string quote_character = 4; // Default: " + string quote_escape_character = 5; // Default: " + } + message JSONOutput { + string record_delimiter = 1; + } + + CSVOutput csv_output = 2; + JSONOutput json_output = 3; + } + + OutputSerialization output_serialization = 5; +} +message QueriedStripe { + bytes records = 1; +} + +message VolumeNeedleStatusRequest { + uint32 volume_id = 1; + uint64 needle_id = 2; +} +message VolumeNeedleStatusResponse { + uint64 needle_id = 1; + uint32 cookie = 2; + uint32 size = 3; + uint64 last_modified = 4; + uint32 crc = 5; + string ttl = 6; +} + +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; +}