Browse Source
seaweed-volume: add Rust project skeleton with proto codegen
seaweed-volume: add Rust project skeleton with proto codegen
Cargo.toml with all dependencies (tonic, axum, clap, prometheus, etc.), build.rs for tonic-build proto compilation, and copies of volume_server.proto, master.proto, remote.proto for gRPC codegen.rust-volume-server
Chris Lu
4 days ago
5 changed files with 1403 additions and 0 deletions
-
81seaweed-volume/Cargo.toml
-
14seaweed-volume/build.rs
-
474seaweed-volume/proto/master.proto
-
76seaweed-volume/proto/remote.proto
-
758seaweed-volume/proto/volume_server.proto
@ -0,0 +1,81 @@ |
|||
[package] |
|||
name = "seaweed-volume" |
|||
version = "0.1.0" |
|||
edition = "2021" |
|||
description = "SeaweedFS Volume Server — Rust implementation" |
|||
|
|||
[dependencies] |
|||
# Async runtime |
|||
tokio = { version = "1", features = ["full"] } |
|||
|
|||
# gRPC + protobuf |
|||
tonic = { version = "0.12", features = ["tls"] } |
|||
prost = "0.13" |
|||
prost-types = "0.13" |
|||
|
|||
# HTTP server |
|||
axum = { version = "0.7", features = ["multipart"] } |
|||
hyper = { version = "1", features = ["full"] } |
|||
hyper-util = { version = "0.1", features = ["tokio"] } |
|||
tower = "0.4" |
|||
tower-http = { version = "0.5", features = ["cors", "trace"] } |
|||
|
|||
# CLI |
|||
clap = { version = "4", features = ["derive"] } |
|||
|
|||
# Metrics |
|||
prometheus = { version = "0.13", features = ["process"] } |
|||
|
|||
# JWT |
|||
jsonwebtoken = "9" |
|||
|
|||
# TLS |
|||
rustls = "0.23" |
|||
tokio-rustls = "0.26" |
|||
rustls-pemfile = "2" |
|||
|
|||
# LevelDB (via RocksDB for better Rust support) |
|||
# Using rusty-leveldb for pure Rust LevelDB |
|||
rusty-leveldb = "3" |
|||
|
|||
# Reed-Solomon erasure coding |
|||
reed-solomon-erasure = "6" |
|||
|
|||
# Logging |
|||
tracing = "0.1" |
|||
tracing-subscriber = { version = "0.3", features = ["env-filter"] } |
|||
|
|||
# Config |
|||
toml = "0.8" |
|||
serde = { version = "1", features = ["derive"] } |
|||
serde_json = "1" |
|||
|
|||
# CRC32 |
|||
crc32fast = "1" |
|||
|
|||
# Memory-mapped files |
|||
memmap2 = "0.9" |
|||
|
|||
# UUID |
|||
uuid = { version = "1", features = ["v4"] } |
|||
|
|||
# HTTP client (for proxying, remote fetch) |
|||
reqwest = { version = "0.12", features = ["rustls-tls", "stream"] } |
|||
|
|||
# Misc |
|||
bytes = "1" |
|||
rand = "0.8" |
|||
chrono = "0.4" |
|||
hex = "0.4" |
|||
parking_lot = "0.12" |
|||
dashmap = "6" |
|||
thiserror = "1" |
|||
anyhow = "1" |
|||
async-trait = "0.1" |
|||
futures = "0.3" |
|||
|
|||
# Disk space checking |
|||
sysinfo = "0.31" |
|||
|
|||
[build-dependencies] |
|||
tonic-build = "0.12" |
|||
@ -0,0 +1,14 @@ |
|||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
tonic_build::configure()
|
|||
.build_server(true)
|
|||
.build_client(true)
|
|||
.compile_protos(
|
|||
&[
|
|||
"proto/volume_server.proto",
|
|||
"proto/master.proto",
|
|||
"proto/remote.proto",
|
|||
],
|
|||
&["proto/"],
|
|||
)?;
|
|||
Ok(())
|
|||
}
|
|||
@ -0,0 +1,474 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package master_pb; |
|||
|
|||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"; |
|||
|
|||
import "volume_server.proto"; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service Seaweed { |
|||
rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { |
|||
} |
|||
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { |
|||
} |
|||
rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { |
|||
} |
|||
rpc Assign (AssignRequest) returns (AssignResponse) { |
|||
} |
|||
rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) { |
|||
} |
|||
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { |
|||
} |
|||
rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) { |
|||
} |
|||
rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) { |
|||
} |
|||
rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) { |
|||
} |
|||
rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) { |
|||
} |
|||
rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) { |
|||
} |
|||
rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) { |
|||
} |
|||
rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) { |
|||
} |
|||
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { |
|||
} |
|||
rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) { |
|||
} |
|||
rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) { |
|||
} |
|||
rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) { |
|||
} |
|||
rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) { |
|||
} |
|||
rpc Ping (PingRequest) returns (PingResponse) { |
|||
} |
|||
rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) { |
|||
} |
|||
rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) { |
|||
} |
|||
rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) { |
|||
} |
|||
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) { |
|||
} |
|||
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) { |
|||
} |
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
message DiskTag { |
|||
uint32 disk_id = 1; |
|||
repeated string tags = 2; |
|||
} |
|||
|
|||
message Heartbeat { |
|||
string ip = 1; |
|||
uint32 port = 2; |
|||
string public_url = 3; |
|||
uint64 max_file_key = 5; |
|||
string data_center = 6; |
|||
string rack = 7; |
|||
uint32 admin_port = 8; |
|||
repeated VolumeInformationMessage volumes = 9; |
|||
// delta volumes |
|||
repeated VolumeShortInformationMessage new_volumes = 10; |
|||
repeated VolumeShortInformationMessage deleted_volumes = 11; |
|||
bool has_no_volumes = 12; |
|||
|
|||
// erasure coding |
|||
repeated VolumeEcShardInformationMessage ec_shards = 16; |
|||
// delta erasure coding shards |
|||
repeated VolumeEcShardInformationMessage new_ec_shards = 17; |
|||
repeated VolumeEcShardInformationMessage deleted_ec_shards = 18; |
|||
bool has_no_ec_shards = 19; |
|||
|
|||
map<string, uint32> max_volume_counts = 4; |
|||
uint32 grpc_port = 20; |
|||
repeated string location_uuids = 21; |
|||
string id = 22; // volume server id, independent of ip:port for stable identification |
|||
|
|||
// state flags |
|||
volume_server_pb.VolumeServerState state = 23; |
|||
|
|||
repeated DiskTag disk_tags = 24; |
|||
} |
|||
|
|||
message HeartbeatResponse { |
|||
uint64 volume_size_limit = 1; |
|||
string leader = 2; |
|||
string metrics_address = 3; |
|||
uint32 metrics_interval_seconds = 4; |
|||
repeated StorageBackend storage_backends = 5; |
|||
repeated string duplicated_uuids = 6; |
|||
bool preallocate = 7; |
|||
} |
|||
|
|||
message VolumeInformationMessage { |
|||
uint32 id = 1; |
|||
uint64 size = 2; |
|||
string collection = 3; |
|||
uint64 file_count = 4; |
|||
uint64 delete_count = 5; |
|||
uint64 deleted_byte_count = 6; |
|||
bool read_only = 7; |
|||
uint32 replica_placement = 8; |
|||
uint32 version = 9; |
|||
uint32 ttl = 10; |
|||
uint32 compact_revision = 11; |
|||
int64 modified_at_second = 12; |
|||
string remote_storage_name = 13; |
|||
string remote_storage_key = 14; |
|||
string disk_type = 15; |
|||
uint32 disk_id = 16; |
|||
} |
|||
|
|||
message VolumeShortInformationMessage { |
|||
uint32 id = 1; |
|||
string collection = 3; |
|||
uint32 replica_placement = 8; |
|||
uint32 version = 9; |
|||
uint32 ttl = 10; |
|||
string disk_type = 15; |
|||
uint32 disk_id = 16; |
|||
} |
|||
|
|||
message VolumeEcShardInformationMessage { |
|||
uint32 id = 1; |
|||
string collection = 2; |
|||
uint32 ec_index_bits = 3; |
|||
string disk_type = 4; |
|||
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume |
|||
uint32 disk_id = 6; |
|||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits |
|||
} |
|||
|
|||
message StorageBackend { |
|||
string type = 1; |
|||
string id = 2; |
|||
map<string, string> properties = 3; |
|||
} |
|||
|
|||
message Empty { |
|||
} |
|||
|
|||
message SuperBlockExtra { |
|||
message ErasureCoding { |
|||
uint32 data = 1; |
|||
uint32 parity = 2; |
|||
repeated uint32 volume_ids = 3; |
|||
} |
|||
ErasureCoding erasure_coding = 1; |
|||
} |
|||
|
|||
message KeepConnectedRequest { |
|||
string client_type = 1; |
|||
string client_address = 3; |
|||
string version = 4; |
|||
string filer_group = 5; |
|||
string data_center = 6; |
|||
string rack = 7; |
|||
} |
|||
|
|||
message VolumeLocation { |
|||
string url = 1; |
|||
string public_url = 2; |
|||
repeated uint32 new_vids = 3; |
|||
repeated uint32 deleted_vids = 4; |
|||
string leader = 5; // optional when leader is not itself |
|||
string data_center = 6; // optional when DataCenter is in use |
|||
uint32 grpc_port = 7; |
|||
repeated uint32 new_ec_vids = 8; |
|||
repeated uint32 deleted_ec_vids = 9; |
|||
} |
|||
|
|||
message ClusterNodeUpdate { |
|||
string node_type = 1; |
|||
string address = 2; |
|||
bool is_add = 4; |
|||
string filer_group = 5; |
|||
int64 created_at_ns = 6; |
|||
} |
|||
|
|||
message KeepConnectedResponse { |
|||
VolumeLocation volume_location = 1; |
|||
ClusterNodeUpdate cluster_node_update = 2; |
|||
} |
|||
|
|||
message LookupVolumeRequest { |
|||
repeated string volume_or_file_ids = 1; |
|||
string collection = 2; // optional, a bit faster if provided. |
|||
} |
|||
message LookupVolumeResponse { |
|||
message VolumeIdLocation { |
|||
string volume_or_file_id = 1; |
|||
repeated Location locations = 2; |
|||
string error = 3; |
|||
string auth = 4; |
|||
} |
|||
repeated VolumeIdLocation volume_id_locations = 1; |
|||
} |
|||
|
|||
message Location { |
|||
string url = 1; |
|||
string public_url = 2; |
|||
uint32 grpc_port = 3; |
|||
string data_center = 4; |
|||
} |
|||
|
|||
message AssignRequest { |
|||
uint64 count = 1; |
|||
string replication = 2; |
|||
string collection = 3; |
|||
string ttl = 4; |
|||
string data_center = 5; |
|||
string rack = 6; |
|||
string data_node = 7; |
|||
uint32 memory_map_max_size_mb = 8; |
|||
uint32 writable_volume_count = 9; |
|||
string disk_type = 10; |
|||
} |
|||
|
|||
message VolumeGrowRequest { |
|||
uint32 writable_volume_count = 1; |
|||
string replication = 2; |
|||
string collection = 3; |
|||
string ttl = 4; |
|||
string data_center = 5; |
|||
string rack = 6; |
|||
string data_node = 7; |
|||
uint32 memory_map_max_size_mb = 8; |
|||
string disk_type = 9; |
|||
} |
|||
|
|||
message AssignResponse { |
|||
string fid = 1; |
|||
uint64 count = 4; |
|||
string error = 5; |
|||
string auth = 6; |
|||
repeated Location replicas = 7; |
|||
Location location = 8; |
|||
} |
|||
|
|||
message StatisticsRequest { |
|||
string replication = 1; |
|||
string collection = 2; |
|||
string ttl = 3; |
|||
string disk_type = 4; |
|||
} |
|||
message StatisticsResponse { |
|||
uint64 total_size = 4; |
|||
uint64 used_size = 5; |
|||
uint64 file_count = 6; |
|||
} |
|||
|
|||
// |
|||
// collection related |
|||
// |
|||
message Collection { |
|||
string name = 1; |
|||
} |
|||
message CollectionListRequest { |
|||
bool include_normal_volumes = 1; |
|||
bool include_ec_volumes = 2; |
|||
} |
|||
message CollectionListResponse { |
|||
repeated Collection collections = 1; |
|||
} |
|||
|
|||
message CollectionDeleteRequest { |
|||
string name = 1; |
|||
} |
|||
message CollectionDeleteResponse { |
|||
} |
|||
|
|||
// |
|||
// volume related |
|||
// |
|||
message DiskInfo { |
|||
string type = 1; |
|||
int64 volume_count = 2; |
|||
int64 max_volume_count = 3; |
|||
int64 free_volume_count = 4; |
|||
int64 active_volume_count = 5; |
|||
repeated VolumeInformationMessage volume_infos = 6; |
|||
repeated VolumeEcShardInformationMessage ec_shard_infos = 7; |
|||
int64 remote_volume_count = 8; |
|||
uint32 disk_id = 9; |
|||
repeated string tags = 10; |
|||
} |
|||
message DataNodeInfo { |
|||
string id = 1; |
|||
map<string, DiskInfo> diskInfos = 2; |
|||
uint32 grpc_port = 3; |
|||
string address = 4; // ip:port for connecting to the volume server |
|||
} |
|||
message RackInfo { |
|||
string id = 1; |
|||
repeated DataNodeInfo data_node_infos = 2; |
|||
map<string, DiskInfo> diskInfos = 3; |
|||
} |
|||
message DataCenterInfo { |
|||
string id = 1; |
|||
repeated RackInfo rack_infos = 2; |
|||
map<string, DiskInfo> diskInfos = 3; |
|||
} |
|||
message TopologyInfo { |
|||
string id = 1; |
|||
repeated DataCenterInfo data_center_infos = 2; |
|||
map<string, DiskInfo> diskInfos = 3; |
|||
} |
|||
message VolumeListRequest { |
|||
} |
|||
message VolumeListResponse { |
|||
TopologyInfo topology_info = 1; |
|||
uint64 volume_size_limit_mb = 2; |
|||
} |
|||
|
|||
message LookupEcVolumeRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message LookupEcVolumeResponse { |
|||
uint32 volume_id = 1; |
|||
message EcShardIdLocation { |
|||
uint32 shard_id = 1; |
|||
repeated Location locations = 2; |
|||
} |
|||
repeated EcShardIdLocation shard_id_locations = 2; |
|||
} |
|||
|
|||
message VacuumVolumeRequest { |
|||
float garbage_threshold = 1; |
|||
uint32 volume_id = 2; |
|||
string collection = 3; |
|||
} |
|||
message VacuumVolumeResponse { |
|||
} |
|||
|
|||
message DisableVacuumRequest { |
|||
} |
|||
message DisableVacuumResponse { |
|||
} |
|||
|
|||
message EnableVacuumRequest { |
|||
} |
|||
message EnableVacuumResponse { |
|||
} |
|||
|
|||
message VolumeMarkReadonlyRequest { |
|||
string ip = 1; |
|||
uint32 port = 2; |
|||
uint32 volume_id = 4; |
|||
string collection = 5; |
|||
uint32 replica_placement = 6; |
|||
uint32 version = 7; |
|||
uint32 ttl = 8; |
|||
string disk_type = 9; |
|||
bool is_readonly = 10; |
|||
} |
|||
message VolumeMarkReadonlyResponse { |
|||
} |
|||
|
|||
message GetMasterConfigurationRequest { |
|||
} |
|||
message GetMasterConfigurationResponse { |
|||
string metrics_address = 1; |
|||
uint32 metrics_interval_seconds = 2; |
|||
repeated StorageBackend storage_backends = 3; |
|||
string default_replication = 4; |
|||
string leader = 5; |
|||
uint32 volume_size_limit_m_b = 6; |
|||
bool volume_preallocate = 7; |
|||
// MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027. |
|||
string maintenance_scripts = 8; |
|||
uint32 maintenance_sleep_minutes = 9; |
|||
} |
|||
|
|||
message ListClusterNodesRequest { |
|||
string client_type = 1; |
|||
string filer_group = 2; |
|||
int32 limit = 4; |
|||
} |
|||
message ListClusterNodesResponse { |
|||
message ClusterNode { |
|||
string address = 1; |
|||
string version = 2; |
|||
int64 created_at_ns = 4; |
|||
string data_center = 5; |
|||
string rack = 6; |
|||
} |
|||
repeated ClusterNode cluster_nodes = 1; |
|||
} |
|||
|
|||
message LeaseAdminTokenRequest { |
|||
int64 previous_token = 1; |
|||
int64 previous_lock_time = 2; |
|||
string lock_name = 3; |
|||
string client_name = 4; |
|||
string message = 5; |
|||
} |
|||
message LeaseAdminTokenResponse { |
|||
int64 token = 1; |
|||
int64 lock_ts_ns = 2; |
|||
} |
|||
|
|||
message ReleaseAdminTokenRequest { |
|||
int64 previous_token = 1; |
|||
int64 previous_lock_time = 2; |
|||
string lock_name = 3; |
|||
} |
|||
message ReleaseAdminTokenResponse { |
|||
} |
|||
|
|||
message PingRequest { |
|||
string target = 1; // default to ping itself |
|||
string target_type = 2; |
|||
} |
|||
message PingResponse { |
|||
int64 start_time_ns = 1; |
|||
int64 remote_time_ns = 2; |
|||
int64 stop_time_ns = 3; |
|||
} |
|||
|
|||
message RaftAddServerRequest { |
|||
string id = 1; |
|||
string address = 2; |
|||
bool voter = 3; |
|||
} |
|||
message RaftAddServerResponse { |
|||
} |
|||
|
|||
message RaftRemoveServerRequest { |
|||
string id = 1; |
|||
bool force = 2; |
|||
} |
|||
message RaftRemoveServerResponse { |
|||
} |
|||
|
|||
message RaftListClusterServersRequest { |
|||
} |
|||
message RaftListClusterServersResponse { |
|||
message ClusterServers { |
|||
string id = 1; |
|||
string address = 2; |
|||
string suffrage = 3; |
|||
bool isLeader = 4; |
|||
} |
|||
repeated ClusterServers cluster_servers = 1; |
|||
} |
|||
|
|||
message RaftLeadershipTransferRequest { |
|||
string target_id = 1; // Optional: target server ID. If empty, transfers to any eligible follower |
|||
string target_address = 2; // Optional: target server address. Required if target_id is specified |
|||
} |
|||
message RaftLeadershipTransferResponse { |
|||
string previous_leader = 1; |
|||
string new_leader = 2; |
|||
} |
|||
|
|||
message VolumeGrowResponse { |
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package remote_pb; |
|||
|
|||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"; |
|||
option java_package = "seaweedfs.client"; |
|||
option java_outer_classname = "FilerProto"; |
|||
|
|||
///////////////////////// |
|||
// Remote Storage related |
|||
///////////////////////// |
|||
message RemoteConf { |
|||
string type = 1; |
|||
string name = 2; |
|||
string s3_access_key = 4; |
|||
string s3_secret_key = 5; |
|||
string s3_region = 6; |
|||
string s3_endpoint = 7; |
|||
string s3_storage_class = 8; |
|||
bool s3_force_path_style = 9; |
|||
bool s3_support_tagging = 13; |
|||
bool s3_v4_signature = 11; |
|||
|
|||
string gcs_google_application_credentials = 10; |
|||
string gcs_project_id = 12; |
|||
|
|||
string azure_account_name = 15; |
|||
string azure_account_key = 16; |
|||
|
|||
string backblaze_key_id = 20; |
|||
string backblaze_application_key = 21; |
|||
string backblaze_endpoint = 22; |
|||
string backblaze_region = 23; |
|||
|
|||
string aliyun_access_key = 25; |
|||
string aliyun_secret_key = 26; |
|||
string aliyun_endpoint = 27; |
|||
string aliyun_region = 28; |
|||
|
|||
string tencent_secret_id = 30; |
|||
string tencent_secret_key = 31; |
|||
string tencent_endpoint = 32; |
|||
|
|||
string baidu_access_key = 35; |
|||
string baidu_secret_key = 36; |
|||
string baidu_endpoint = 37; |
|||
string baidu_region = 38; |
|||
|
|||
string wasabi_access_key = 40; |
|||
string wasabi_secret_key = 41; |
|||
string wasabi_endpoint = 42; |
|||
string wasabi_region = 43; |
|||
|
|||
string filebase_access_key = 60; |
|||
string filebase_secret_key = 61; |
|||
string filebase_endpoint = 62; |
|||
|
|||
string storj_access_key = 65; |
|||
string storj_secret_key = 66; |
|||
string storj_endpoint = 67; |
|||
|
|||
string contabo_access_key = 68; |
|||
string contabo_secret_key = 69; |
|||
string contabo_endpoint = 70; |
|||
string contabo_region = 71; |
|||
} |
|||
|
|||
message RemoteStorageMapping { |
|||
map<string,RemoteStorageLocation> mappings = 1; |
|||
string primary_bucket_storage_name = 2; |
|||
} |
|||
message RemoteStorageLocation { |
|||
string name = 1; |
|||
string bucket = 2; |
|||
string path = 3; |
|||
} |
|||
@ -0,0 +1,758 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package volume_server_pb; |
|||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"; |
|||
|
|||
import "remote.proto"; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
// Persistent state for volume servers. |
|||
message VolumeServerState { |
|||
// whether the server is in maintenance (i.e. read-only) mode. |
|||
bool maintenance = 1; |
|||
// incremental version counter |
|||
uint32 version = 2; |
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service VolumeServer { |
|||
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. |
|||
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) { |
|||
} |
|||
|
|||
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) { |
|||
} |
|||
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) { |
|||
} |
|||
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) { |
|||
} |
|||
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) { |
|||
} |
|||
|
|||
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) { |
|||
} |
|||
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) { |
|||
} |
|||
|
|||
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) { |
|||
} |
|||
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) { |
|||
} |
|||
|
|||
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) { |
|||
} |
|||
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) { |
|||
} |
|||
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) { |
|||
} |
|||
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) { |
|||
} |
|||
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) { |
|||
} |
|||
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) { |
|||
} |
|||
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) { |
|||
} |
|||
|
|||
rpc GetState (GetStateRequest) returns (GetStateResponse) { |
|||
} |
|||
rpc SetState (SetStateRequest) returns (SetStateResponse) { |
|||
} |
|||
|
|||
// copy the .idx .dat files, and mount this volume |
|||
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) { |
|||
} |
|||
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) { |
|||
} |
|||
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) { |
|||
} |
|||
rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) { |
|||
} |
|||
|
|||
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) { |
|||
} |
|||
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) { |
|||
} |
|||
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) { |
|||
} |
|||
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) { |
|||
} |
|||
|
|||
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) { |
|||
} |
|||
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) { |
|||
} |
|||
|
|||
// erasure coding |
|||
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) { |
|||
} |
|||
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) { |
|||
} |
|||
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) { |
|||
} |
|||
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) { |
|||
} |
|||
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) { |
|||
} |
|||
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) { |
|||
} |
|||
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) { |
|||
} |
|||
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) { |
|||
} |
|||
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) { |
|||
} |
|||
rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) { |
|||
} |
|||
|
|||
// tiered storage |
|||
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) { |
|||
} |
|||
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) { |
|||
} |
|||
|
|||
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) { |
|||
} |
|||
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) { |
|||
} |
|||
|
|||
// remote storage |
|||
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) { |
|||
} |
|||
|
|||
// scrubbing |
|||
rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) { |
|||
} |
|||
rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) { |
|||
} |
|||
|
|||
// <experimental> query |
|||
rpc Query (QueryRequest) returns (stream QueriedStripe) { |
|||
} |
|||
|
|||
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) { |
|||
} |
|||
|
|||
rpc Ping (PingRequest) returns (PingResponse) { |
|||
} |
|||
|
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
message BatchDeleteRequest { |
|||
repeated string file_ids = 1; |
|||
bool skip_cookie_check = 2; |
|||
} |
|||
|
|||
message BatchDeleteResponse { |
|||
repeated DeleteResult results = 1; |
|||
} |
|||
message DeleteResult { |
|||
string file_id = 1; |
|||
int32 status = 2; |
|||
string error = 3; |
|||
uint32 size = 4; |
|||
uint32 version = 5; |
|||
} |
|||
|
|||
message Empty { |
|||
} |
|||
|
|||
message VacuumVolumeCheckRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VacuumVolumeCheckResponse { |
|||
double garbage_ratio = 1; |
|||
} |
|||
|
|||
message VacuumVolumeCompactRequest { |
|||
uint32 volume_id = 1; |
|||
int64 preallocate = 2; |
|||
} |
|||
message VacuumVolumeCompactResponse { |
|||
int64 processed_bytes = 1; |
|||
float load_avg_1m = 2; |
|||
} |
|||
|
|||
message VacuumVolumeCommitRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VacuumVolumeCommitResponse { |
|||
bool is_read_only = 1; |
|||
uint64 volume_size = 2; |
|||
} |
|||
|
|||
message VacuumVolumeCleanupRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VacuumVolumeCleanupResponse { |
|||
} |
|||
|
|||
message DeleteCollectionRequest { |
|||
string collection = 1; |
|||
} |
|||
message DeleteCollectionResponse { |
|||
} |
|||
|
|||
message AllocateVolumeRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
int64 preallocate = 3; |
|||
string replication = 4; |
|||
string ttl = 5; |
|||
uint32 memory_map_max_size_mb = 6; |
|||
string disk_type = 7; |
|||
uint32 version = 8; |
|||
} |
|||
message AllocateVolumeResponse { |
|||
} |
|||
|
|||
message VolumeSyncStatusRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeSyncStatusResponse { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
string replication = 4; |
|||
string ttl = 5; |
|||
uint64 tail_offset = 6; |
|||
uint32 compact_revision = 7; |
|||
uint64 idx_file_size = 8; |
|||
uint32 version = 9; |
|||
} |
|||
|
|||
message VolumeIncrementalCopyRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 since_ns = 2; |
|||
} |
|||
message VolumeIncrementalCopyResponse { |
|||
bytes file_content = 1; |
|||
} |
|||
|
|||
message VolumeMountRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeMountResponse { |
|||
} |
|||
|
|||
message VolumeUnmountRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeUnmountResponse { |
|||
} |
|||
|
|||
message VolumeDeleteRequest { |
|||
uint32 volume_id = 1; |
|||
bool only_empty = 2; |
|||
} |
|||
message VolumeDeleteResponse { |
|||
} |
|||
|
|||
message VolumeMarkReadonlyRequest { |
|||
uint32 volume_id = 1; |
|||
bool persist = 2; |
|||
} |
|||
message VolumeMarkReadonlyResponse { |
|||
} |
|||
|
|||
message VolumeMarkWritableRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeMarkWritableResponse { |
|||
} |
|||
|
|||
message VolumeConfigureRequest { |
|||
uint32 volume_id = 1; |
|||
string replication = 2; |
|||
} |
|||
message VolumeConfigureResponse { |
|||
string error = 1; |
|||
} |
|||
|
|||
message VolumeStatusRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeStatusResponse { |
|||
bool is_read_only = 1; |
|||
uint64 volume_size = 2; |
|||
uint64 file_count = 3; |
|||
uint64 file_deleted_count = 4; |
|||
} |
|||
|
|||
message GetStateRequest { |
|||
} |
|||
message GetStateResponse { |
|||
VolumeServerState state = 1; |
|||
} |
|||
|
|||
message SetStateRequest { |
|||
// SetState updates *all* volume server flags at once. Retrieve state with GetState(), |
|||
// modify individual flags as required, then call this RPC to update. |
|||
VolumeServerState state = 1; |
|||
} |
|||
message SetStateResponse { |
|||
VolumeServerState state = 1; |
|||
} |
|||
|
|||
message VolumeCopyRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
string replication = 3; |
|||
string ttl = 4; |
|||
string source_data_node = 5; |
|||
string disk_type = 6; |
|||
int64 io_byte_per_second = 7; |
|||
} |
|||
message VolumeCopyResponse { |
|||
uint64 last_append_at_ns = 1; |
|||
int64 processed_bytes = 2; |
|||
} |
|||
|
|||
message CopyFileRequest { |
|||
uint32 volume_id = 1; |
|||
string ext = 2; |
|||
uint32 compaction_revision = 3; |
|||
uint64 stop_offset = 4; |
|||
string collection = 5; |
|||
bool is_ec_volume = 6; |
|||
bool ignore_source_file_not_found = 7; |
|||
} |
|||
message CopyFileResponse { |
|||
bytes file_content = 1; |
|||
int64 modified_ts_ns = 2; |
|||
} |
|||
|
|||
message ReceiveFileRequest { |
|||
oneof data { |
|||
ReceiveFileInfo info = 1; |
|||
bytes file_content = 2; |
|||
} |
|||
} |
|||
|
|||
message ReceiveFileInfo { |
|||
uint32 volume_id = 1; |
|||
string ext = 2; |
|||
string collection = 3; |
|||
bool is_ec_volume = 4; |
|||
uint32 shard_id = 5; |
|||
uint64 file_size = 6; |
|||
} |
|||
|
|||
message ReceiveFileResponse { |
|||
uint64 bytes_written = 1; |
|||
string error = 2; |
|||
} |
|||
|
|||
message ReadNeedleBlobRequest { |
|||
uint32 volume_id = 1; |
|||
int64 offset = 3; // actual offset |
|||
int32 size = 4; |
|||
} |
|||
message ReadNeedleBlobResponse { |
|||
bytes needle_blob = 1; |
|||
} |
|||
|
|||
message ReadNeedleMetaRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 needle_id = 2; |
|||
int64 offset = 3; // actual offset |
|||
int32 size = 4; |
|||
} |
|||
message ReadNeedleMetaResponse { |
|||
uint32 cookie = 1; |
|||
uint64 last_modified = 2; |
|||
uint32 crc = 3; |
|||
string ttl = 4; |
|||
uint64 append_at_ns = 5; |
|||
} |
|||
|
|||
message WriteNeedleBlobRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 needle_id = 2; |
|||
int32 size = 3; |
|||
bytes needle_blob = 4; |
|||
} |
|||
message WriteNeedleBlobResponse { |
|||
} |
|||
|
|||
message ReadAllNeedlesRequest { |
|||
repeated uint32 volume_ids = 1; |
|||
} |
|||
message ReadAllNeedlesResponse { |
|||
uint32 volume_id = 1; |
|||
uint64 needle_id = 2; |
|||
uint32 cookie = 3; |
|||
bytes needle_blob = 5; |
|||
bool needle_blob_compressed = 6; |
|||
uint64 last_modified = 7; |
|||
uint32 crc = 8; |
|||
bytes name = 9; |
|||
bytes mime = 10; |
|||
} |
|||
|
|||
message VolumeTailSenderRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 since_ns = 2; |
|||
uint32 idle_timeout_seconds = 3; |
|||
} |
|||
message VolumeTailSenderResponse { |
|||
bytes needle_header = 1; |
|||
bytes needle_body = 2; |
|||
bool is_last_chunk = 3; |
|||
uint32 version = 4; |
|||
} |
|||
|
|||
message VolumeTailReceiverRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 since_ns = 2; |
|||
uint32 idle_timeout_seconds = 3; |
|||
string source_volume_server = 4; |
|||
} |
|||
message VolumeTailReceiverResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsGenerateRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
} |
|||
message VolumeEcShardsGenerateResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsRebuildRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
} |
|||
message VolumeEcShardsRebuildResponse { |
|||
repeated uint32 rebuilt_shard_ids = 1; |
|||
} |
|||
|
|||
message VolumeEcShardsCopyRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
repeated uint32 shard_ids = 3; |
|||
bool copy_ecx_file = 4; |
|||
string source_data_node = 5; |
|||
bool copy_ecj_file = 6; |
|||
bool copy_vif_file = 7; |
|||
uint32 disk_id = 8; // Target disk ID for storing EC shards |
|||
} |
|||
message VolumeEcShardsCopyResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsDeleteRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
repeated uint32 shard_ids = 3; |
|||
} |
|||
message VolumeEcShardsDeleteResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsMountRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
repeated uint32 shard_ids = 3; |
|||
} |
|||
message VolumeEcShardsMountResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsUnmountRequest { |
|||
uint32 volume_id = 1; |
|||
repeated uint32 shard_ids = 3; |
|||
} |
|||
message VolumeEcShardsUnmountResponse { |
|||
} |
|||
|
|||
message VolumeEcShardReadRequest { |
|||
uint32 volume_id = 1; |
|||
uint32 shard_id = 2; |
|||
int64 offset = 3; |
|||
int64 size = 4; |
|||
uint64 file_key = 5; |
|||
} |
|||
message VolumeEcShardReadResponse { |
|||
bytes data = 1; |
|||
bool is_deleted = 2; |
|||
} |
|||
|
|||
message VolumeEcBlobDeleteRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
uint64 file_key = 3; |
|||
uint32 version = 4; |
|||
} |
|||
message VolumeEcBlobDeleteResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsToVolumeRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
} |
|||
message VolumeEcShardsToVolumeResponse { |
|||
} |
|||
|
|||
message VolumeEcShardsInfoRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message VolumeEcShardsInfoResponse { |
|||
repeated EcShardInfo ec_shard_infos = 1; |
|||
uint64 volume_size = 2; |
|||
uint64 file_count = 3; |
|||
uint64 file_deleted_count = 4; |
|||
} |
|||
|
|||
message EcShardInfo { |
|||
uint32 shard_id = 1; |
|||
int64 size = 2; |
|||
string collection = 3; |
|||
uint32 volume_id = 4; |
|||
} |
|||
|
|||
message ReadVolumeFileStatusRequest { |
|||
uint32 volume_id = 1; |
|||
} |
|||
message ReadVolumeFileStatusResponse { |
|||
uint32 volume_id = 1; |
|||
uint64 idx_file_timestamp_seconds = 2; |
|||
uint64 idx_file_size = 3; |
|||
uint64 dat_file_timestamp_seconds = 4; |
|||
uint64 dat_file_size = 5; |
|||
uint64 file_count = 6; |
|||
uint32 compaction_revision = 7; |
|||
string collection = 8; |
|||
string disk_type = 9; |
|||
VolumeInfo volume_info = 10; |
|||
uint32 version = 11; |
|||
} |
|||
|
|||
message DiskStatus { |
|||
string dir = 1; |
|||
uint64 all = 2; |
|||
uint64 used = 3; |
|||
uint64 free = 4; |
|||
float percent_free = 5; |
|||
float percent_used = 6; |
|||
string disk_type = 7; |
|||
} |
|||
|
|||
message MemStatus { |
|||
int32 goroutines = 1; |
|||
uint64 all = 2; |
|||
uint64 used = 3; |
|||
uint64 free = 4; |
|||
uint64 self = 5; |
|||
uint64 heap = 6; |
|||
uint64 stack = 7; |
|||
} |
|||
|
|||
// tired storage on volume servers |
|||
message RemoteFile { |
|||
string backend_type = 1; |
|||
string backend_id = 2; |
|||
string key = 3; |
|||
uint64 offset = 4; |
|||
uint64 file_size = 5; |
|||
uint64 modified_time = 6; |
|||
string extension = 7; |
|||
} |
|||
message VolumeInfo { |
|||
repeated RemoteFile files = 1; |
|||
uint32 version = 2; |
|||
string replication = 3; |
|||
uint32 bytes_offset = 4; |
|||
int64 dat_file_size = 5; // store the original dat file size |
|||
uint64 expire_at_sec = 6; // expiration time of ec volume |
|||
bool read_only = 7; |
|||
EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4) |
|||
} |
|||
|
|||
// EcShardConfig specifies erasure coding shard configuration |
|||
message EcShardConfig { |
|||
uint32 data_shards = 1; // Number of data shards (e.g., 10) |
|||
uint32 parity_shards = 2; // Number of parity shards (e.g., 4) |
|||
} |
|||
message OldVersionVolumeInfo { |
|||
repeated RemoteFile files = 1; |
|||
uint32 version = 2; |
|||
string replication = 3; |
|||
uint32 BytesOffset = 4; |
|||
int64 dat_file_size = 5; // store the original dat file size |
|||
uint64 DestroyTime = 6; // expiration time of ec volume |
|||
bool read_only = 7; |
|||
} |
|||
|
|||
// tiered storage |
|||
message VolumeTierMoveDatToRemoteRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
string destination_backend_name = 3; |
|||
bool keep_local_dat_file = 4; |
|||
} |
|||
message VolumeTierMoveDatToRemoteResponse { |
|||
int64 processed = 1; |
|||
float processedPercentage = 2; |
|||
} |
|||
|
|||
message VolumeTierMoveDatFromRemoteRequest { |
|||
uint32 volume_id = 1; |
|||
string collection = 2; |
|||
bool keep_remote_dat_file = 3; |
|||
} |
|||
message VolumeTierMoveDatFromRemoteResponse { |
|||
int64 processed = 1; |
|||
float processedPercentage = 2; |
|||
} |
|||
|
|||
message VolumeServerStatusRequest { |
|||
|
|||
} |
|||
message VolumeServerStatusResponse { |
|||
repeated DiskStatus disk_statuses = 1; |
|||
MemStatus memory_status = 2; |
|||
string version = 3; |
|||
string data_center = 4; |
|||
string rack = 5; |
|||
VolumeServerState state = 6; |
|||
} |
|||
|
|||
message VolumeServerLeaveRequest { |
|||
} |
|||
message VolumeServerLeaveResponse { |
|||
} |
|||
|
|||
// remote storage |
|||
message FetchAndWriteNeedleRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 needle_id = 2; |
|||
uint32 cookie = 3; |
|||
int64 offset = 4; |
|||
int64 size = 5; |
|||
message Replica { |
|||
string url = 1; |
|||
string public_url = 2; |
|||
int32 grpc_port = 3; |
|||
} |
|||
repeated Replica replicas = 6; |
|||
string auth = 7; |
|||
// remote conf |
|||
remote_pb.RemoteConf remote_conf = 15; |
|||
remote_pb.RemoteStorageLocation remote_location = 16; |
|||
} |
|||
message FetchAndWriteNeedleResponse { |
|||
string e_tag = 1; |
|||
} |
|||
|
|||
enum VolumeScrubMode { |
|||
UNKNOWN = 0; |
|||
INDEX = 1; |
|||
FULL = 2; |
|||
LOCAL = 3; |
|||
} |
|||
|
|||
message ScrubVolumeRequest { |
|||
VolumeScrubMode mode = 1; |
|||
// optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed. |
|||
repeated uint32 volume_ids = 2; |
|||
} |
|||
message ScrubVolumeResponse { |
|||
uint64 total_volumes = 1; |
|||
uint64 total_files = 2; |
|||
repeated uint32 broken_volume_ids = 3; |
|||
repeated string details = 4; |
|||
} |
|||
|
|||
message ScrubEcVolumeRequest { |
|||
VolumeScrubMode mode = 1; |
|||
// optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed. |
|||
repeated uint32 volume_ids = 2; |
|||
} |
|||
message ScrubEcVolumeResponse { |
|||
uint64 total_volumes = 1; |
|||
uint64 total_files = 2; |
|||
repeated uint32 broken_volume_ids = 3; |
|||
repeated EcShardInfo broken_shard_infos = 4; |
|||
repeated string details = 5; |
|||
} |
|||
|
|||
// select on volume servers |
|||
message QueryRequest { |
|||
repeated string selections = 1; |
|||
repeated string from_file_ids = 2; |
|||
message Filter { |
|||
string field = 1; |
|||
string operand = 2; |
|||
string value = 3; |
|||
} |
|||
Filter filter = 3; |
|||
|
|||
message InputSerialization { |
|||
// NONE | GZIP | BZIP2 |
|||
string compression_type = 1; |
|||
message CSVInput { |
|||
string file_header_info = 1; // Valid values: NONE | USE | IGNORE |
|||
string record_delimiter = 2; // Default: \n |
|||
string field_delimiter = 3; // Default: , |
|||
string quote_character = 4; // Default: " |
|||
string quote_escape_character = 5; // Default: " |
|||
string comments = 6; // Default: # |
|||
// If true, records might contain record delimiters within quote characters |
|||
bool allow_quoted_record_delimiter = 7; // default False. |
|||
} |
|||
message JSONInput { |
|||
string type = 1; // Valid values: DOCUMENT | LINES |
|||
} |
|||
message ParquetInput { |
|||
} |
|||
|
|||
CSVInput csv_input = 2; |
|||
JSONInput json_input = 3; |
|||
ParquetInput parquet_input = 4; |
|||
} |
|||
InputSerialization input_serialization = 4; |
|||
|
|||
message OutputSerialization { |
|||
message CSVOutput { |
|||
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED |
|||
string record_delimiter = 2; // Default: \n |
|||
string field_delimiter = 3; // Default: , |
|||
string quote_character = 4; // Default: " |
|||
string quote_escape_character = 5; // Default: " |
|||
} |
|||
message JSONOutput { |
|||
string record_delimiter = 1; |
|||
} |
|||
|
|||
CSVOutput csv_output = 2; |
|||
JSONOutput json_output = 3; |
|||
} |
|||
|
|||
OutputSerialization output_serialization = 5; |
|||
} |
|||
message QueriedStripe { |
|||
bytes records = 1; |
|||
} |
|||
|
|||
message VolumeNeedleStatusRequest { |
|||
uint32 volume_id = 1; |
|||
uint64 needle_id = 2; |
|||
} |
|||
message VolumeNeedleStatusResponse { |
|||
uint64 needle_id = 1; |
|||
uint32 cookie = 2; |
|||
uint32 size = 3; |
|||
uint64 last_modified = 4; |
|||
uint32 crc = 5; |
|||
string ttl = 6; |
|||
} |
|||
|
|||
message PingRequest { |
|||
string target = 1; // default to ping itself |
|||
string target_type = 2; |
|||
} |
|||
message PingResponse { |
|||
int64 start_time_ns = 1; |
|||
int64 remote_time_ns = 2; |
|||
int64 stop_time_ns = 3; |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue