Browse Source
seaweed-volume: add Rust project skeleton with proto codegen
seaweed-volume: add Rust project skeleton with proto codegen
Cargo.toml with all dependencies (tonic, axum, clap, prometheus, etc.), build.rs for tonic-build proto compilation, and copies of volume_server.proto, master.proto, remote.proto for gRPC codegen.rust-volume-server
Chris Lu
4 days ago
5 changed files with 1403 additions and 0 deletions
-
81seaweed-volume/Cargo.toml
-
14seaweed-volume/build.rs
-
474seaweed-volume/proto/master.proto
-
76seaweed-volume/proto/remote.proto
-
758seaweed-volume/proto/volume_server.proto
@ -0,0 +1,81 @@ |
|||||
|
[package] |
||||
|
name = "seaweed-volume" |
||||
|
version = "0.1.0" |
||||
|
edition = "2021" |
||||
|
description = "SeaweedFS Volume Server — Rust implementation" |
||||
|
|
||||
|
[dependencies] |
||||
|
# Async runtime |
||||
|
tokio = { version = "1", features = ["full"] } |
||||
|
|
||||
|
# gRPC + protobuf |
||||
|
tonic = { version = "0.12", features = ["tls"] } |
||||
|
prost = "0.13" |
||||
|
prost-types = "0.13" |
||||
|
|
||||
|
# HTTP server |
||||
|
axum = { version = "0.7", features = ["multipart"] } |
||||
|
hyper = { version = "1", features = ["full"] } |
||||
|
hyper-util = { version = "0.1", features = ["tokio"] } |
||||
|
tower = "0.4" |
||||
|
tower-http = { version = "0.5", features = ["cors", "trace"] } |
||||
|
|
||||
|
# CLI |
||||
|
clap = { version = "4", features = ["derive"] } |
||||
|
|
||||
|
# Metrics |
||||
|
prometheus = { version = "0.13", features = ["process"] } |
||||
|
|
||||
|
# JWT |
||||
|
jsonwebtoken = "9" |
||||
|
|
||||
|
# TLS |
||||
|
rustls = "0.23" |
||||
|
tokio-rustls = "0.26" |
||||
|
rustls-pemfile = "2" |
||||
|
|
||||
|
# LevelDB (via RocksDB for better Rust support) |
||||
|
# Using rusty-leveldb for pure Rust LevelDB |
||||
|
rusty-leveldb = "3" |
||||
|
|
||||
|
# Reed-Solomon erasure coding |
||||
|
reed-solomon-erasure = "6" |
||||
|
|
||||
|
# Logging |
||||
|
tracing = "0.1" |
||||
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] } |
||||
|
|
||||
|
# Config |
||||
|
toml = "0.8" |
||||
|
serde = { version = "1", features = ["derive"] } |
||||
|
serde_json = "1" |
||||
|
|
||||
|
# CRC32 |
||||
|
crc32fast = "1" |
||||
|
|
||||
|
# Memory-mapped files |
||||
|
memmap2 = "0.9" |
||||
|
|
||||
|
# UUID |
||||
|
uuid = { version = "1", features = ["v4"] } |
||||
|
|
||||
|
# HTTP client (for proxying, remote fetch) |
||||
|
reqwest = { version = "0.12", features = ["rustls-tls", "stream"] } |
||||
|
|
||||
|
# Misc |
||||
|
bytes = "1" |
||||
|
rand = "0.8" |
||||
|
chrono = "0.4" |
||||
|
hex = "0.4" |
||||
|
parking_lot = "0.12" |
||||
|
dashmap = "6" |
||||
|
thiserror = "1" |
||||
|
anyhow = "1" |
||||
|
async-trait = "0.1" |
||||
|
futures = "0.3" |
||||
|
|
||||
|
# Disk space checking |
||||
|
sysinfo = "0.31" |
||||
|
|
||||
|
[build-dependencies] |
||||
|
tonic-build = "0.12" |
||||
@ -0,0 +1,14 @@ |
|||||
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
tonic_build::configure()
|
||||
|
.build_server(true)
|
||||
|
.build_client(true)
|
||||
|
.compile_protos(
|
||||
|
&[
|
||||
|
"proto/volume_server.proto",
|
||||
|
"proto/master.proto",
|
||||
|
"proto/remote.proto",
|
||||
|
],
|
||||
|
&["proto/"],
|
||||
|
)?;
|
||||
|
Ok(())
|
||||
|
}
|
||||
@ -0,0 +1,474 @@ |
|||||
|
syntax = "proto3"; |
||||
|
|
||||
|
package master_pb; |
||||
|
|
||||
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"; |
||||
|
|
||||
|
import "volume_server.proto"; |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
service Seaweed { |
||||
|
rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { |
||||
|
} |
||||
|
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { |
||||
|
} |
||||
|
rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { |
||||
|
} |
||||
|
rpc Assign (AssignRequest) returns (AssignResponse) { |
||||
|
} |
||||
|
rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) { |
||||
|
} |
||||
|
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { |
||||
|
} |
||||
|
rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { |
||||
|
} |
||||
|
rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) { |
||||
|
} |
||||
|
rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) { |
||||
|
} |
||||
|
rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) { |
||||
|
} |
||||
|
rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) { |
||||
|
} |
||||
|
rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) { |
||||
|
} |
||||
|
rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) { |
||||
|
} |
||||
|
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { |
||||
|
} |
||||
|
rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) { |
||||
|
} |
||||
|
rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) { |
||||
|
} |
||||
|
rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) { |
||||
|
} |
||||
|
rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) { |
||||
|
} |
||||
|
rpc Ping (PingRequest) returns (PingResponse) { |
||||
|
} |
||||
|
rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) { |
||||
|
} |
||||
|
rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) { |
||||
|
} |
||||
|
rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) { |
||||
|
} |
||||
|
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) { |
||||
|
} |
||||
|
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
message DiskTag { |
||||
|
uint32 disk_id = 1; |
||||
|
repeated string tags = 2; |
||||
|
} |
||||
|
|
||||
|
message Heartbeat { |
||||
|
string ip = 1; |
||||
|
uint32 port = 2; |
||||
|
string public_url = 3; |
||||
|
uint64 max_file_key = 5; |
||||
|
string data_center = 6; |
||||
|
string rack = 7; |
||||
|
uint32 admin_port = 8; |
||||
|
repeated VolumeInformationMessage volumes = 9; |
||||
|
// delta volumes |
||||
|
repeated VolumeShortInformationMessage new_volumes = 10; |
||||
|
repeated VolumeShortInformationMessage deleted_volumes = 11; |
||||
|
bool has_no_volumes = 12; |
||||
|
|
||||
|
// erasure coding |
||||
|
repeated VolumeEcShardInformationMessage ec_shards = 16; |
||||
|
// delta erasure coding shards |
||||
|
repeated VolumeEcShardInformationMessage new_ec_shards = 17; |
||||
|
repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; |
||||
|
bool has_no_ec_shards = 19; |
||||
|
|
||||
|
map<string, uint32> max_volume_counts = 4; |
||||
|
uint32 grpc_port = 20; |
||||
|
repeated string location_uuids = 21; |
||||
|
string id = 22; // volume server id, independent of ip:port for stable identification |
||||
|
|
||||
|
// state flags |
||||
|
volume_server_pb.VolumeServerState state = 23; |
||||
|
|
||||
|
repeated DiskTag disk_tags = 24; |
||||
|
} |
||||
|
|
||||
|
message HeartbeatResponse { |
||||
|
uint64 volume_size_limit = 1; |
||||
|
string leader = 2; |
||||
|
string metrics_address = 3; |
||||
|
uint32 metrics_interval_seconds = 4; |
||||
|
repeated StorageBackend storage_backends = 5; |
||||
|
repeated string duplicated_uuids = 6; |
||||
|
bool preallocate = 7; |
||||
|
} |
||||
|
|
||||
|
message VolumeInformationMessage { |
||||
|
uint32 id = 1; |
||||
|
uint64 size = 2; |
||||
|
string collection = 3; |
||||
|
uint64 file_count = 4; |
||||
|
uint64 delete_count = 5; |
||||
|
uint64 deleted_byte_count = 6; |
||||
|
bool read_only = 7; |
||||
|
uint32 replica_placement = 8; |
||||
|
uint32 version = 9; |
||||
|
uint32 ttl = 10; |
||||
|
uint32 compact_revision = 11; |
||||
|
int64 modified_at_second = 12; |
||||
|
string remote_storage_name = 13; |
||||
|
string remote_storage_key = 14; |
||||
|
string disk_type = 15; |
||||
|
uint32 disk_id = 16; |
||||
|
} |
||||
|
|
||||
|
message VolumeShortInformationMessage { |
||||
|
uint32 id = 1; |
||||
|
string collection = 3; |
||||
|
uint32 replica_placement = 8; |
||||
|
uint32 version = 9; |
||||
|
uint32 ttl = 10; |
||||
|
string disk_type = 15; |
||||
|
uint32 disk_id = 16; |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardInformationMessage { |
||||
|
uint32 id = 1; |
||||
|
string collection = 2; |
||||
|
uint32 ec_index_bits = 3; |
||||
|
string disk_type = 4; |
||||
|
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume |
||||
|
uint32 disk_id = 6; |
||||
|
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits |
||||
|
} |
||||
|
|
||||
|
message StorageBackend { |
||||
|
string type = 1; |
||||
|
string id = 2; |
||||
|
map<string, string> properties = 3; |
||||
|
} |
||||
|
|
||||
|
message Empty { |
||||
|
} |
||||
|
|
||||
|
message SuperBlockExtra { |
||||
|
message ErasureCoding { |
||||
|
uint32 data = 1; |
||||
|
uint32 parity = 2; |
||||
|
repeated uint32 volume_ids = 3; |
||||
|
} |
||||
|
ErasureCoding erasure_coding = 1; |
||||
|
} |
||||
|
|
||||
|
message KeepConnectedRequest { |
||||
|
string client_type = 1; |
||||
|
string client_address = 3; |
||||
|
string version = 4; |
||||
|
string filer_group = 5; |
||||
|
string data_center = 6; |
||||
|
string rack = 7; |
||||
|
} |
||||
|
|
||||
|
message VolumeLocation { |
||||
|
string url = 1; |
||||
|
string public_url = 2; |
||||
|
repeated uint32 new_vids = 3; |
||||
|
repeated uint32 deleted_vids = 4; |
||||
|
string leader = 5; // optional when leader is not itself |
||||
|
string data_center = 6; // optional when DataCenter is in use |
||||
|
uint32 grpc_port = 7; |
||||
|
repeated uint32 new_ec_vids = 8; |
||||
|
repeated uint32 deleted_ec_vids = 9; |
||||
|
} |
||||
|
|
||||
|
message ClusterNodeUpdate { |
||||
|
string node_type = 1; |
||||
|
string address = 2; |
||||
|
bool is_add = 4; |
||||
|
string filer_group = 5; |
||||
|
int64 created_at_ns = 6; |
||||
|
} |
||||
|
|
||||
|
message KeepConnectedResponse { |
||||
|
VolumeLocation volume_location = 1; |
||||
|
ClusterNodeUpdate cluster_node_update = 2; |
||||
|
} |
||||
|
|
||||
|
message LookupVolumeRequest { |
||||
|
repeated string volume_or_file_ids = 1; |
||||
|
string collection = 2; // optional, a bit faster if provided. |
||||
|
} |
||||
|
message LookupVolumeResponse { |
||||
|
message VolumeIdLocation { |
||||
|
string volume_or_file_id = 1; |
||||
|
repeated Location locations = 2; |
||||
|
string error = 3; |
||||
|
string auth = 4; |
||||
|
} |
||||
|
repeated VolumeIdLocation volume_id_locations = 1; |
||||
|
} |
||||
|
|
||||
|
message Location { |
||||
|
string url = 1; |
||||
|
string public_url = 2; |
||||
|
uint32 grpc_port = 3; |
||||
|
string data_center = 4; |
||||
|
} |
||||
|
|
||||
|
message AssignRequest { |
||||
|
uint64 count = 1; |
||||
|
string replication = 2; |
||||
|
string collection = 3; |
||||
|
string ttl = 4; |
||||
|
string data_center = 5; |
||||
|
string rack = 6; |
||||
|
string data_node = 7; |
||||
|
uint32 memory_map_max_size_mb = 8; |
||||
|
uint32 writable_volume_count = 9; |
||||
|
string disk_type = 10; |
||||
|
} |
||||
|
|
||||
|
message VolumeGrowRequest { |
||||
|
uint32 writable_volume_count = 1; |
||||
|
string replication = 2; |
||||
|
string collection = 3; |
||||
|
string ttl = 4; |
||||
|
string data_center = 5; |
||||
|
string rack = 6; |
||||
|
string data_node = 7; |
||||
|
uint32 memory_map_max_size_mb = 8; |
||||
|
string disk_type = 9; |
||||
|
} |
||||
|
|
||||
|
message AssignResponse { |
||||
|
string fid = 1; |
||||
|
uint64 count = 4; |
||||
|
string error = 5; |
||||
|
string auth = 6; |
||||
|
repeated Location replicas = 7; |
||||
|
Location location = 8; |
||||
|
} |
||||
|
|
||||
|
message StatisticsRequest { |
||||
|
string replication = 1; |
||||
|
string collection = 2; |
||||
|
string ttl = 3; |
||||
|
string disk_type = 4; |
||||
|
} |
||||
|
message StatisticsResponse { |
||||
|
uint64 total_size = 4; |
||||
|
uint64 used_size = 5; |
||||
|
uint64 file_count = 6; |
||||
|
} |
||||
|
|
||||
|
// |
||||
|
// collection related |
||||
|
// |
||||
|
message Collection { |
||||
|
string name = 1; |
||||
|
} |
||||
|
message CollectionListRequest { |
||||
|
bool include_normal_volumes = 1; |
||||
|
bool include_ec_volumes = 2; |
||||
|
} |
||||
|
message CollectionListResponse { |
||||
|
repeated Collection collections = 1; |
||||
|
} |
||||
|
|
||||
|
message CollectionDeleteRequest { |
||||
|
string name = 1; |
||||
|
} |
||||
|
message CollectionDeleteResponse { |
||||
|
} |
||||
|
|
||||
|
// |
||||
|
// volume related |
||||
|
// |
||||
|
message DiskInfo { |
||||
|
string type = 1; |
||||
|
int64 volume_count = 2; |
||||
|
int64 max_volume_count = 3; |
||||
|
int64 free_volume_count = 4; |
||||
|
int64 active_volume_count = 5; |
||||
|
repeated VolumeInformationMessage volume_infos = 6; |
||||
|
repeated VolumeEcShardInformationMessage ec_shard_infos = 7; |
||||
|
int64 remote_volume_count = 8; |
||||
|
uint32 disk_id = 9; |
||||
|
repeated string tags = 10; |
||||
|
} |
||||
|
message DataNodeInfo { |
||||
|
string id = 1; |
||||
|
map<string, DiskInfo> diskInfos = 2; |
||||
|
uint32 grpc_port = 3; |
||||
|
string address = 4; // ip:port for connecting to the volume server |
||||
|
} |
||||
|
message RackInfo { |
||||
|
string id = 1; |
||||
|
repeated DataNodeInfo data_node_infos = 2; |
||||
|
map<string, DiskInfo> diskInfos = 3; |
||||
|
} |
||||
|
message DataCenterInfo { |
||||
|
string id = 1; |
||||
|
repeated RackInfo rack_infos = 2; |
||||
|
map<string, DiskInfo> diskInfos = 3; |
||||
|
} |
||||
|
message TopologyInfo { |
||||
|
string id = 1; |
||||
|
repeated DataCenterInfo data_center_infos = 2; |
||||
|
map<string, DiskInfo> diskInfos = 3; |
||||
|
} |
||||
|
message VolumeListRequest { |
||||
|
} |
||||
|
message VolumeListResponse { |
||||
|
TopologyInfo topology_info = 1; |
||||
|
uint64 volume_size_limit_mb = 2; |
||||
|
} |
||||
|
|
||||
|
message LookupEcVolumeRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message LookupEcVolumeResponse { |
||||
|
uint32 volume_id = 1; |
||||
|
message EcShardIdLocation { |
||||
|
uint32 shard_id = 1; |
||||
|
repeated Location locations = 2; |
||||
|
} |
||||
|
repeated EcShardIdLocation shard_id_locations = 2; |
||||
|
} |
||||
|
|
||||
|
message VacuumVolumeRequest { |
||||
|
float garbage_threshold = 1; |
||||
|
uint32 volume_id = 2; |
||||
|
string collection = 3; |
||||
|
} |
||||
|
message VacuumVolumeResponse { |
||||
|
} |
||||
|
|
||||
|
message DisableVacuumRequest { |
||||
|
} |
||||
|
message DisableVacuumResponse { |
||||
|
} |
||||
|
|
||||
|
message EnableVacuumRequest { |
||||
|
} |
||||
|
message EnableVacuumResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeMarkReadonlyRequest { |
||||
|
string ip = 1; |
||||
|
uint32 port = 2; |
||||
|
uint32 volume_id = 4; |
||||
|
string collection = 5; |
||||
|
uint32 replica_placement = 6; |
||||
|
uint32 version = 7; |
||||
|
uint32 ttl = 8; |
||||
|
string disk_type = 9; |
||||
|
bool is_readonly = 10; |
||||
|
} |
||||
|
message VolumeMarkReadonlyResponse { |
||||
|
} |
||||
|
|
||||
|
message GetMasterConfigurationRequest { |
||||
|
} |
||||
|
message GetMasterConfigurationResponse { |
||||
|
string metrics_address = 1; |
||||
|
uint32 metrics_interval_seconds = 2; |
||||
|
repeated StorageBackend storage_backends = 3; |
||||
|
string default_replication = 4; |
||||
|
string leader = 5; |
||||
|
uint32 volume_size_limit_m_b = 6; |
||||
|
bool volume_preallocate = 7; |
||||
|
// MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027. |
||||
|
string maintenance_scripts = 8; |
||||
|
uint32 maintenance_sleep_minutes = 9; |
||||
|
} |
||||
|
|
||||
|
message ListClusterNodesRequest { |
||||
|
string client_type = 1; |
||||
|
string filer_group = 2; |
||||
|
int32 limit = 4; |
||||
|
} |
||||
|
message ListClusterNodesResponse { |
||||
|
message ClusterNode { |
||||
|
string address = 1; |
||||
|
string version = 2; |
||||
|
int64 created_at_ns = 4; |
||||
|
string data_center = 5; |
||||
|
string rack = 6; |
||||
|
} |
||||
|
repeated ClusterNode cluster_nodes = 1; |
||||
|
} |
||||
|
|
||||
|
message LeaseAdminTokenRequest { |
||||
|
int64 previous_token = 1; |
||||
|
int64 previous_lock_time = 2; |
||||
|
string lock_name = 3; |
||||
|
string client_name = 4; |
||||
|
string message = 5; |
||||
|
} |
||||
|
message LeaseAdminTokenResponse { |
||||
|
int64 token = 1; |
||||
|
int64 lock_ts_ns = 2; |
||||
|
} |
||||
|
|
||||
|
message ReleaseAdminTokenRequest { |
||||
|
int64 previous_token = 1; |
||||
|
int64 previous_lock_time = 2; |
||||
|
string lock_name = 3; |
||||
|
} |
||||
|
message ReleaseAdminTokenResponse { |
||||
|
} |
||||
|
|
||||
|
message PingRequest { |
||||
|
string target = 1; // default to ping itself |
||||
|
string target_type = 2; |
||||
|
} |
||||
|
message PingResponse { |
||||
|
int64 start_time_ns = 1; |
||||
|
int64 remote_time_ns = 2; |
||||
|
int64 stop_time_ns = 3; |
||||
|
} |
||||
|
|
||||
|
message RaftAddServerRequest { |
||||
|
string id = 1; |
||||
|
string address = 2; |
||||
|
bool voter = 3; |
||||
|
} |
||||
|
message RaftAddServerResponse { |
||||
|
} |
||||
|
|
||||
|
message RaftRemoveServerRequest { |
||||
|
string id = 1; |
||||
|
bool force = 2; |
||||
|
} |
||||
|
message RaftRemoveServerResponse { |
||||
|
} |
||||
|
|
||||
|
message RaftListClusterServersRequest { |
||||
|
} |
||||
|
message RaftListClusterServersResponse { |
||||
|
message ClusterServers { |
||||
|
string id = 1; |
||||
|
string address = 2; |
||||
|
string suffrage = 3; |
||||
|
bool isLeader = 4; |
||||
|
} |
||||
|
repeated ClusterServers cluster_servers = 1; |
||||
|
} |
||||
|
|
||||
|
message RaftLeadershipTransferRequest { |
||||
|
string target_id = 1; // Optional: target server ID. If empty, transfers to any eligible follower |
||||
|
string target_address = 2; // Optional: target server address. Required if target_id is specified |
||||
|
} |
||||
|
message RaftLeadershipTransferResponse { |
||||
|
string previous_leader = 1; |
||||
|
string new_leader = 2; |
||||
|
} |
||||
|
|
||||
|
message VolumeGrowResponse { |
||||
|
} |
||||
@ -0,0 +1,76 @@ |
|||||
|
syntax = "proto3"; |
||||
|
|
||||
|
package remote_pb; |
||||
|
|
||||
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"; |
||||
|
option java_package = "seaweedfs.client"; |
||||
|
option java_outer_classname = "FilerProto"; |
||||
|
|
||||
|
///////////////////////// |
||||
|
// Remote Storage related |
||||
|
///////////////////////// |
||||
|
message RemoteConf { |
||||
|
string type = 1; |
||||
|
string name = 2; |
||||
|
string s3_access_key = 4; |
||||
|
string s3_secret_key = 5; |
||||
|
string s3_region = 6; |
||||
|
string s3_endpoint = 7; |
||||
|
string s3_storage_class = 8; |
||||
|
bool s3_force_path_style = 9; |
||||
|
bool s3_support_tagging = 13; |
||||
|
bool s3_v4_signature = 11; |
||||
|
|
||||
|
string gcs_google_application_credentials = 10; |
||||
|
string gcs_project_id = 12; |
||||
|
|
||||
|
string azure_account_name = 15; |
||||
|
string azure_account_key = 16; |
||||
|
|
||||
|
string backblaze_key_id = 20; |
||||
|
string backblaze_application_key = 21; |
||||
|
string backblaze_endpoint = 22; |
||||
|
string backblaze_region = 23; |
||||
|
|
||||
|
string aliyun_access_key = 25; |
||||
|
string aliyun_secret_key = 26; |
||||
|
string aliyun_endpoint = 27; |
||||
|
string aliyun_region = 28; |
||||
|
|
||||
|
string tencent_secret_id = 30; |
||||
|
string tencent_secret_key = 31; |
||||
|
string tencent_endpoint = 32; |
||||
|
|
||||
|
string baidu_access_key = 35; |
||||
|
string baidu_secret_key = 36; |
||||
|
string baidu_endpoint = 37; |
||||
|
string baidu_region = 38; |
||||
|
|
||||
|
string wasabi_access_key = 40; |
||||
|
string wasabi_secret_key = 41; |
||||
|
string wasabi_endpoint = 42; |
||||
|
string wasabi_region = 43; |
||||
|
|
||||
|
string filebase_access_key = 60; |
||||
|
string filebase_secret_key = 61; |
||||
|
string filebase_endpoint = 62; |
||||
|
|
||||
|
string storj_access_key = 65; |
||||
|
string storj_secret_key = 66; |
||||
|
string storj_endpoint = 67; |
||||
|
|
||||
|
string contabo_access_key = 68; |
||||
|
string contabo_secret_key = 69; |
||||
|
string contabo_endpoint = 70; |
||||
|
string contabo_region = 71; |
||||
|
} |
||||
|
|
||||
|
message RemoteStorageMapping { |
||||
|
map<string,RemoteStorageLocation> mappings = 1; |
||||
|
string primary_bucket_storage_name = 2; |
||||
|
} |
||||
|
message RemoteStorageLocation { |
||||
|
string name = 1; |
||||
|
string bucket = 2; |
||||
|
string path = 3; |
||||
|
} |
||||
@ -0,0 +1,758 @@ |
|||||
|
syntax = "proto3"; |
||||
|
|
||||
|
package volume_server_pb; |
||||
|
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"; |
||||
|
|
||||
|
import "remote.proto"; |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
// Persistent state for volume servers. |
||||
|
message VolumeServerState { |
||||
|
// whether the server is in maintenance (i.e. read-only) mode. |
||||
|
bool maintenance = 1; |
||||
|
// incremental version counter |
||||
|
uint32 version = 2; |
||||
|
} |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
service VolumeServer { |
||||
|
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. |
||||
|
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) { |
||||
|
} |
||||
|
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) { |
||||
|
} |
||||
|
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) { |
||||
|
} |
||||
|
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) { |
||||
|
} |
||||
|
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) { |
||||
|
} |
||||
|
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) { |
||||
|
} |
||||
|
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) { |
||||
|
} |
||||
|
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) { |
||||
|
} |
||||
|
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { |
||||
|
} |
||||
|
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) { |
||||
|
} |
||||
|
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) { |
||||
|
} |
||||
|
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc GetState (GetStateRequest) returns (GetStateResponse) { |
||||
|
} |
||||
|
rpc SetState (SetStateRequest) returns (SetStateResponse) { |
||||
|
} |
||||
|
|
||||
|
// copy the .idx .dat files, and mount this volume |
||||
|
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) { |
||||
|
} |
||||
|
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) { |
||||
|
} |
||||
|
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) { |
||||
|
} |
||||
|
rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) { |
||||
|
} |
||||
|
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) { |
||||
|
} |
||||
|
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) { |
||||
|
} |
||||
|
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) { |
||||
|
} |
||||
|
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) { |
||||
|
} |
||||
|
|
||||
|
// erasure coding |
||||
|
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) { |
||||
|
} |
||||
|
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) { |
||||
|
} |
||||
|
rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) { |
||||
|
} |
||||
|
|
||||
|
// tiered storage |
||||
|
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) { |
||||
|
} |
||||
|
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) { |
||||
|
} |
||||
|
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) { |
||||
|
} |
||||
|
|
||||
|
// remote storage |
||||
|
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) { |
||||
|
} |
||||
|
|
||||
|
// scrubbing |
||||
|
rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) { |
||||
|
} |
||||
|
rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) { |
||||
|
} |
||||
|
|
||||
|
// <experimental> query |
||||
|
rpc Query (QueryRequest) returns (stream QueriedStripe) { |
||||
|
} |
||||
|
|
||||
|
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc Ping (PingRequest) returns (PingResponse) { |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
message BatchDeleteRequest { |
||||
|
repeated string file_ids = 1; |
||||
|
bool skip_cookie_check = 2; |
||||
|
} |
||||
|
|
||||
|
message BatchDeleteResponse { |
||||
|
repeated DeleteResult results = 1; |
||||
|
} |
||||
|
message DeleteResult { |
||||
|
string file_id = 1; |
||||
|
int32 status = 2; |
||||
|
string error = 3; |
||||
|
uint32 size = 4; |
||||
|
uint32 version = 5; |
||||
|
} |
||||
|
|
||||
|
message Empty { |
||||
|
} |
||||
|
|
||||
|
message VacuumVolumeCheckRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VacuumVolumeCheckResponse { |
||||
|
double garbage_ratio = 1; |
||||
|
} |
||||
|
|
||||
|
message VacuumVolumeCompactRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
int64 preallocate = 2; |
||||
|
} |
||||
|
message VacuumVolumeCompactResponse { |
||||
|
int64 processed_bytes = 1; |
||||
|
float load_avg_1m = 2; |
||||
|
} |
||||
|
|
||||
|
message VacuumVolumeCommitRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VacuumVolumeCommitResponse { |
||||
|
bool is_read_only = 1; |
||||
|
uint64 volume_size = 2; |
||||
|
} |
||||
|
|
||||
|
message VacuumVolumeCleanupRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VacuumVolumeCleanupResponse { |
||||
|
} |
||||
|
|
||||
|
message DeleteCollectionRequest { |
||||
|
string collection = 1; |
||||
|
} |
||||
|
message DeleteCollectionResponse { |
||||
|
} |
||||
|
|
||||
|
message AllocateVolumeRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
int64 preallocate = 3; |
||||
|
string replication = 4; |
||||
|
string ttl = 5; |
||||
|
uint32 memory_map_max_size_mb = 6; |
||||
|
string disk_type = 7; |
||||
|
uint32 version = 8; |
||||
|
} |
||||
|
message AllocateVolumeResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeSyncStatusRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeSyncStatusResponse { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
string replication = 4; |
||||
|
string ttl = 5; |
||||
|
uint64 tail_offset = 6; |
||||
|
uint32 compact_revision = 7; |
||||
|
uint64 idx_file_size = 8; |
||||
|
uint32 version = 9; |
||||
|
} |
||||
|
|
||||
|
message VolumeIncrementalCopyRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 since_ns = 2; |
||||
|
} |
||||
|
message VolumeIncrementalCopyResponse { |
||||
|
bytes file_content = 1; |
||||
|
} |
||||
|
|
||||
|
message VolumeMountRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeMountResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeUnmountRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeUnmountResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeDeleteRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
bool only_empty = 2; |
||||
|
} |
||||
|
message VolumeDeleteResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeMarkReadonlyRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
bool persist = 2; |
||||
|
} |
||||
|
message VolumeMarkReadonlyResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeMarkWritableRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeMarkWritableResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeConfigureRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string replication = 2; |
||||
|
} |
||||
|
message VolumeConfigureResponse { |
||||
|
string error = 1; |
||||
|
} |
||||
|
|
||||
|
message VolumeStatusRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeStatusResponse { |
||||
|
bool is_read_only = 1; |
||||
|
uint64 volume_size = 2; |
||||
|
uint64 file_count = 3; |
||||
|
uint64 file_deleted_count = 4; |
||||
|
} |
||||
|
|
||||
|
message GetStateRequest { |
||||
|
} |
||||
|
message GetStateResponse { |
||||
|
VolumeServerState state = 1; |
||||
|
} |
||||
|
|
||||
|
message SetStateRequest { |
||||
|
// SetState updates *all* volume server flags at once. Retrieve state with GetState(), |
||||
|
// modify individual flags as required, then call this RPC to update. |
||||
|
VolumeServerState state = 1; |
||||
|
} |
||||
|
message SetStateResponse { |
||||
|
VolumeServerState state = 1; |
||||
|
} |
||||
|
|
||||
|
message VolumeCopyRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
string replication = 3; |
||||
|
string ttl = 4; |
||||
|
string source_data_node = 5; |
||||
|
string disk_type = 6; |
||||
|
int64 io_byte_per_second = 7; |
||||
|
} |
||||
|
message VolumeCopyResponse { |
||||
|
uint64 last_append_at_ns = 1; |
||||
|
int64 processed_bytes = 2; |
||||
|
} |
||||
|
|
||||
|
message CopyFileRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string ext = 2; |
||||
|
uint32 compaction_revision = 3; |
||||
|
uint64 stop_offset = 4; |
||||
|
string collection = 5; |
||||
|
bool is_ec_volume = 6; |
||||
|
bool ignore_source_file_not_found = 7; |
||||
|
} |
||||
|
message CopyFileResponse { |
||||
|
bytes file_content = 1; |
||||
|
int64 modified_ts_ns = 2; |
||||
|
} |
||||
|
|
||||
|
message ReceiveFileRequest { |
||||
|
oneof data { |
||||
|
ReceiveFileInfo info = 1; |
||||
|
bytes file_content = 2; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
message ReceiveFileInfo { |
||||
|
uint32 volume_id = 1; |
||||
|
string ext = 2; |
||||
|
string collection = 3; |
||||
|
bool is_ec_volume = 4; |
||||
|
uint32 shard_id = 5; |
||||
|
uint64 file_size = 6; |
||||
|
} |
||||
|
|
||||
|
message ReceiveFileResponse { |
||||
|
uint64 bytes_written = 1; |
||||
|
string error = 2; |
||||
|
} |
||||
|
|
||||
|
message ReadNeedleBlobRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
int64 offset = 3; // actual offset |
||||
|
int32 size = 4; |
||||
|
} |
||||
|
message ReadNeedleBlobResponse { |
||||
|
bytes needle_blob = 1; |
||||
|
} |
||||
|
|
||||
|
message ReadNeedleMetaRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 needle_id = 2; |
||||
|
int64 offset = 3; // actual offset |
||||
|
int32 size = 4; |
||||
|
} |
||||
|
message ReadNeedleMetaResponse { |
||||
|
uint32 cookie = 1; |
||||
|
uint64 last_modified = 2; |
||||
|
uint32 crc = 3; |
||||
|
string ttl = 4; |
||||
|
uint64 append_at_ns = 5; |
||||
|
} |
||||
|
|
||||
|
message WriteNeedleBlobRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 needle_id = 2; |
||||
|
int32 size = 3; |
||||
|
bytes needle_blob = 4; |
||||
|
} |
||||
|
message WriteNeedleBlobResponse { |
||||
|
} |
||||
|
|
||||
|
message ReadAllNeedlesRequest { |
||||
|
repeated uint32 volume_ids = 1; |
||||
|
} |
||||
|
message ReadAllNeedlesResponse { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 needle_id = 2; |
||||
|
uint32 cookie = 3; |
||||
|
bytes needle_blob = 5; |
||||
|
bool needle_blob_compressed = 6; |
||||
|
uint64 last_modified = 7; |
||||
|
uint32 crc = 8; |
||||
|
bytes name = 9; |
||||
|
bytes mime = 10; |
||||
|
} |
||||
|
|
||||
|
message VolumeTailSenderRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 since_ns = 2; |
||||
|
uint32 idle_timeout_seconds = 3; |
||||
|
} |
||||
|
message VolumeTailSenderResponse { |
||||
|
bytes needle_header = 1; |
||||
|
bytes needle_body = 2; |
||||
|
bool is_last_chunk = 3; |
||||
|
uint32 version = 4; |
||||
|
} |
||||
|
|
||||
|
message VolumeTailReceiverRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 since_ns = 2; |
||||
|
uint32 idle_timeout_seconds = 3; |
||||
|
string source_volume_server = 4; |
||||
|
} |
||||
|
message VolumeTailReceiverResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsGenerateRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
} |
||||
|
message VolumeEcShardsGenerateResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsRebuildRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
} |
||||
|
message VolumeEcShardsRebuildResponse { |
||||
|
repeated uint32 rebuilt_shard_ids = 1; |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsCopyRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
repeated uint32 shard_ids = 3; |
||||
|
bool copy_ecx_file = 4; |
||||
|
string source_data_node = 5; |
||||
|
bool copy_ecj_file = 6; |
||||
|
bool copy_vif_file = 7; |
||||
|
uint32 disk_id = 8; // Target disk ID for storing EC shards |
||||
|
} |
||||
|
message VolumeEcShardsCopyResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsDeleteRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
repeated uint32 shard_ids = 3; |
||||
|
} |
||||
|
message VolumeEcShardsDeleteResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsMountRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
repeated uint32 shard_ids = 3; |
||||
|
} |
||||
|
message VolumeEcShardsMountResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsUnmountRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
repeated uint32 shard_ids = 3; |
||||
|
} |
||||
|
message VolumeEcShardsUnmountResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardReadRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint32 shard_id = 2; |
||||
|
int64 offset = 3; |
||||
|
int64 size = 4; |
||||
|
uint64 file_key = 5; |
||||
|
} |
||||
|
message VolumeEcShardReadResponse { |
||||
|
bytes data = 1; |
||||
|
bool is_deleted = 2; |
||||
|
} |
||||
|
|
||||
|
message VolumeEcBlobDeleteRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
uint64 file_key = 3; |
||||
|
uint32 version = 4; |
||||
|
} |
||||
|
message VolumeEcBlobDeleteResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsToVolumeRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
} |
||||
|
message VolumeEcShardsToVolumeResponse { |
||||
|
} |
||||
|
|
||||
|
message VolumeEcShardsInfoRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message VolumeEcShardsInfoResponse { |
||||
|
repeated EcShardInfo ec_shard_infos = 1; |
||||
|
uint64 volume_size = 2; |
||||
|
uint64 file_count = 3; |
||||
|
uint64 file_deleted_count = 4; |
||||
|
} |
||||
|
|
||||
|
message EcShardInfo { |
||||
|
uint32 shard_id = 1; |
||||
|
int64 size = 2; |
||||
|
string collection = 3; |
||||
|
uint32 volume_id = 4; |
||||
|
} |
||||
|
|
||||
|
message ReadVolumeFileStatusRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
} |
||||
|
message ReadVolumeFileStatusResponse { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 idx_file_timestamp_seconds = 2; |
||||
|
uint64 idx_file_size = 3; |
||||
|
uint64 dat_file_timestamp_seconds = 4; |
||||
|
uint64 dat_file_size = 5; |
||||
|
uint64 file_count = 6; |
||||
|
uint32 compaction_revision = 7; |
||||
|
string collection = 8; |
||||
|
string disk_type = 9; |
||||
|
VolumeInfo volume_info = 10; |
||||
|
uint32 version = 11; |
||||
|
} |
||||
|
|
||||
|
message DiskStatus { |
||||
|
string dir = 1; |
||||
|
uint64 all = 2; |
||||
|
uint64 used = 3; |
||||
|
uint64 free = 4; |
||||
|
float percent_free = 5; |
||||
|
float percent_used = 6; |
||||
|
string disk_type = 7; |
||||
|
} |
||||
|
|
||||
|
message MemStatus { |
||||
|
int32 goroutines = 1; |
||||
|
uint64 all = 2; |
||||
|
uint64 used = 3; |
||||
|
uint64 free = 4; |
||||
|
uint64 self = 5; |
||||
|
uint64 heap = 6; |
||||
|
uint64 stack = 7; |
||||
|
} |
||||
|
|
||||
|
// tired storage on volume servers |
||||
|
message RemoteFile { |
||||
|
string backend_type = 1; |
||||
|
string backend_id = 2; |
||||
|
string key = 3; |
||||
|
uint64 offset = 4; |
||||
|
uint64 file_size = 5; |
||||
|
uint64 modified_time = 6; |
||||
|
string extension = 7; |
||||
|
} |
||||
|
message VolumeInfo { |
||||
|
repeated RemoteFile files = 1; |
||||
|
uint32 version = 2; |
||||
|
string replication = 3; |
||||
|
uint32 bytes_offset = 4; |
||||
|
int64 dat_file_size = 5; // store the original dat file size |
||||
|
uint64 expire_at_sec = 6; // expiration time of ec volume |
||||
|
bool read_only = 7; |
||||
|
EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4) |
||||
|
} |
||||
|
|
||||
|
// EcShardConfig specifies erasure coding shard configuration |
||||
|
message EcShardConfig { |
||||
|
uint32 data_shards = 1; // Number of data shards (e.g., 10) |
||||
|
uint32 parity_shards = 2; // Number of parity shards (e.g., 4) |
||||
|
} |
||||
|
message OldVersionVolumeInfo { |
||||
|
repeated RemoteFile files = 1; |
||||
|
uint32 version = 2; |
||||
|
string replication = 3; |
||||
|
uint32 BytesOffset = 4; |
||||
|
int64 dat_file_size = 5; // store the original dat file size |
||||
|
uint64 DestroyTime = 6; // expiration time of ec volume |
||||
|
bool read_only = 7; |
||||
|
} |
||||
|
|
||||
|
// tiered storage |
||||
|
message VolumeTierMoveDatToRemoteRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
string destination_backend_name = 3; |
||||
|
bool keep_local_dat_file = 4; |
||||
|
} |
||||
|
message VolumeTierMoveDatToRemoteResponse { |
||||
|
int64 processed = 1; |
||||
|
float processedPercentage = 2; |
||||
|
} |
||||
|
|
||||
|
message VolumeTierMoveDatFromRemoteRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
string collection = 2; |
||||
|
bool keep_remote_dat_file = 3; |
||||
|
} |
||||
|
message VolumeTierMoveDatFromRemoteResponse { |
||||
|
int64 processed = 1; |
||||
|
float processedPercentage = 2; |
||||
|
} |
||||
|
|
||||
|
message VolumeServerStatusRequest { |
||||
|
|
||||
|
} |
||||
|
message VolumeServerStatusResponse { |
||||
|
repeated DiskStatus disk_statuses = 1; |
||||
|
MemStatus memory_status = 2; |
||||
|
string version = 3; |
||||
|
string data_center = 4; |
||||
|
string rack = 5; |
||||
|
VolumeServerState state = 6; |
||||
|
} |
||||
|
|
||||
|
message VolumeServerLeaveRequest { |
||||
|
} |
||||
|
message VolumeServerLeaveResponse { |
||||
|
} |
||||
|
|
||||
|
// remote storage |
||||
|
message FetchAndWriteNeedleRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 needle_id = 2; |
||||
|
uint32 cookie = 3; |
||||
|
int64 offset = 4; |
||||
|
int64 size = 5; |
||||
|
message Replica { |
||||
|
string url = 1; |
||||
|
string public_url = 2; |
||||
|
int32 grpc_port = 3; |
||||
|
} |
||||
|
repeated Replica replicas = 6; |
||||
|
string auth = 7; |
||||
|
// remote conf |
||||
|
remote_pb.RemoteConf remote_conf = 15; |
||||
|
remote_pb.RemoteStorageLocation remote_location = 16; |
||||
|
} |
||||
|
message FetchAndWriteNeedleResponse { |
||||
|
string e_tag = 1; |
||||
|
} |
||||
|
|
||||
|
enum VolumeScrubMode { |
||||
|
UNKNOWN = 0; |
||||
|
INDEX = 1; |
||||
|
FULL = 2; |
||||
|
LOCAL = 3; |
||||
|
} |
||||
|
|
||||
|
message ScrubVolumeRequest { |
||||
|
VolumeScrubMode mode = 1; |
||||
|
// optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed. |
||||
|
repeated uint32 volume_ids = 2; |
||||
|
} |
||||
|
message ScrubVolumeResponse { |
||||
|
uint64 total_volumes = 1; |
||||
|
uint64 total_files = 2; |
||||
|
repeated uint32 broken_volume_ids = 3; |
||||
|
repeated string details = 4; |
||||
|
} |
||||
|
|
||||
|
message ScrubEcVolumeRequest { |
||||
|
VolumeScrubMode mode = 1; |
||||
|
// optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed. |
||||
|
repeated uint32 volume_ids = 2; |
||||
|
} |
||||
|
message ScrubEcVolumeResponse { |
||||
|
uint64 total_volumes = 1; |
||||
|
uint64 total_files = 2; |
||||
|
repeated uint32 broken_volume_ids = 3; |
||||
|
repeated EcShardInfo broken_shard_infos = 4; |
||||
|
repeated string details = 5; |
||||
|
} |
||||
|
|
||||
|
// select on volume servers |
||||
|
message QueryRequest { |
||||
|
repeated string selections = 1; |
||||
|
repeated string from_file_ids = 2; |
||||
|
message Filter { |
||||
|
string field = 1; |
||||
|
string operand = 2; |
||||
|
string value = 3; |
||||
|
} |
||||
|
Filter filter = 3; |
||||
|
|
||||
|
message InputSerialization { |
||||
|
// NONE | GZIP | BZIP2 |
||||
|
string compression_type = 1; |
||||
|
message CSVInput { |
||||
|
string file_header_info = 1; // Valid values: NONE | USE | IGNORE |
||||
|
string record_delimiter = 2; // Default: \n |
||||
|
string field_delimiter = 3; // Default: , |
||||
|
string quote_character = 4; // Default: " |
||||
|
string quote_escape_character = 5; // Default: " |
||||
|
string comments = 6; // Default: # |
||||
|
// If true, records might contain record delimiters within quote characters |
||||
|
bool allow_quoted_record_delimiter = 7; // default False. |
||||
|
} |
||||
|
message JSONInput { |
||||
|
string type = 1; // Valid values: DOCUMENT | LINES |
||||
|
} |
||||
|
message ParquetInput { |
||||
|
} |
||||
|
|
||||
|
CSVInput csv_input = 2; |
||||
|
JSONInput json_input = 3; |
||||
|
ParquetInput parquet_input = 4; |
||||
|
} |
||||
|
InputSerialization input_serialization = 4; |
||||
|
|
||||
|
message OutputSerialization { |
||||
|
message CSVOutput { |
||||
|
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED |
||||
|
string record_delimiter = 2; // Default: \n |
||||
|
string field_delimiter = 3; // Default: , |
||||
|
string quote_character = 4; // Default: " |
||||
|
string quote_escape_character = 5; // Default: " |
||||
|
} |
||||
|
message JSONOutput { |
||||
|
string record_delimiter = 1; |
||||
|
} |
||||
|
|
||||
|
CSVOutput csv_output = 2; |
||||
|
JSONOutput json_output = 3; |
||||
|
} |
||||
|
|
||||
|
OutputSerialization output_serialization = 5; |
||||
|
} |
||||
|
message QueriedStripe { |
||||
|
bytes records = 1; |
||||
|
} |
||||
|
|
||||
|
message VolumeNeedleStatusRequest { |
||||
|
uint32 volume_id = 1; |
||||
|
uint64 needle_id = 2; |
||||
|
} |
||||
|
message VolumeNeedleStatusResponse { |
||||
|
uint64 needle_id = 1; |
||||
|
uint32 cookie = 2; |
||||
|
uint32 size = 3; |
||||
|
uint64 last_modified = 4; |
||||
|
uint32 crc = 5; |
||||
|
string ttl = 6; |
||||
|
} |
||||
|
|
||||
|
message PingRequest { |
||||
|
string target = 1; // default to ping itself |
||||
|
string target_type = 2; |
||||
|
} |
||||
|
message PingResponse { |
||||
|
int64 start_time_ns = 1; |
||||
|
int64 remote_time_ns = 2; |
||||
|
int64 stop_time_ns = 3; |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue