You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

483 lines
12 KiB

6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. //////////////////////////////////////////////////
  4. service VolumeServer {
  5. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  6. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  7. }
  8. rpc FileGet (FileGetRequest) returns (stream FileGetResponse) {
  9. }
  10. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  11. }
  12. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
  13. }
  14. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  15. }
  16. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  17. }
  18. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  19. }
  20. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  21. }
  22. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  23. }
  24. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  25. }
  26. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  27. }
  28. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  29. }
  30. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  31. }
  32. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  33. }
  34. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  35. }
  36. // copy the .idx .dat files, and mount this volume
  37. rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
  38. }
  39. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  40. }
  41. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  42. }
  43. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  44. }
  45. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  46. }
  47. // erasure coding
  48. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  49. }
  50. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  51. }
  52. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  53. }
  54. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  55. }
  56. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  57. }
  58. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  59. }
  60. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  61. }
  62. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  63. }
  64. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  65. }
  66. // tiered storage
  67. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  68. }
  69. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  70. }
  71. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  72. }
  73. // <experimental> query
  74. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  75. }
  76. }
  77. //////////////////////////////////////////////////
  78. message BatchDeleteRequest {
  79. repeated string file_ids = 1;
  80. bool skip_cookie_check = 2;
  81. }
  82. message BatchDeleteResponse {
  83. repeated DeleteResult results = 1;
  84. }
  85. message DeleteResult {
  86. string file_id = 1;
  87. int32 status = 2;
  88. string error = 3;
  89. uint32 size = 4;
  90. uint32 version = 5;
  91. }
  92. message FileGetRequest {
  93. string file_id = 1;
  94. bool accept_gzip = 2;
  95. }
  96. message FileGetResponse {
  97. bytes data = 1;
  98. uint32 content_length = 2;
  99. string content_type = 3;
  100. uint64 last_modified = 4;
  101. string filename = 5;
  102. string etag = 6;
  103. bool is_gzipped = 7;
  104. map<string, string> headers = 8;
  105. int32 errorCode = 9;
  106. }
  107. message Empty {
  108. }
  109. message VacuumVolumeCheckRequest {
  110. uint32 volume_id = 1;
  111. }
  112. message VacuumVolumeCheckResponse {
  113. double garbage_ratio = 1;
  114. }
  115. message VacuumVolumeCompactRequest {
  116. uint32 volume_id = 1;
  117. int64 preallocate = 2;
  118. }
  119. message VacuumVolumeCompactResponse {
  120. }
  121. message VacuumVolumeCommitRequest {
  122. uint32 volume_id = 1;
  123. }
  124. message VacuumVolumeCommitResponse {
  125. bool is_read_only = 1;
  126. }
  127. message VacuumVolumeCleanupRequest {
  128. uint32 volume_id = 1;
  129. }
  130. message VacuumVolumeCleanupResponse {
  131. }
  132. message DeleteCollectionRequest {
  133. string collection = 1;
  134. }
  135. message DeleteCollectionResponse {
  136. }
  137. message AllocateVolumeRequest {
  138. uint32 volume_id = 1;
  139. string collection = 2;
  140. int64 preallocate = 3;
  141. string replication = 4;
  142. string ttl = 5;
  143. uint32 memory_map_max_size_mb = 6;
  144. }
  145. message AllocateVolumeResponse {
  146. }
  147. message VolumeSyncStatusRequest {
  148. uint32 volume_id = 1;
  149. }
  150. message VolumeSyncStatusResponse {
  151. uint32 volume_id = 1;
  152. string collection = 2;
  153. string replication = 4;
  154. string ttl = 5;
  155. uint64 tail_offset = 6;
  156. uint32 compact_revision = 7;
  157. uint64 idx_file_size = 8;
  158. }
  159. message VolumeIncrementalCopyRequest {
  160. uint32 volume_id = 1;
  161. uint64 since_ns = 2;
  162. }
  163. message VolumeIncrementalCopyResponse {
  164. bytes file_content = 1;
  165. }
  166. message VolumeMountRequest {
  167. uint32 volume_id = 1;
  168. }
  169. message VolumeMountResponse {
  170. }
  171. message VolumeUnmountRequest {
  172. uint32 volume_id = 1;
  173. }
  174. message VolumeUnmountResponse {
  175. }
  176. message VolumeDeleteRequest {
  177. uint32 volume_id = 1;
  178. }
  179. message VolumeDeleteResponse {
  180. }
  181. message VolumeMarkReadonlyRequest {
  182. uint32 volume_id = 1;
  183. }
  184. message VolumeMarkReadonlyResponse {
  185. }
  186. message VolumeConfigureRequest {
  187. uint32 volume_id = 1;
  188. string replication = 2;
  189. }
  190. message VolumeConfigureResponse {
  191. string error = 1;
  192. }
  193. message VolumeCopyRequest {
  194. uint32 volume_id = 1;
  195. string collection = 2;
  196. string replication = 3;
  197. string ttl = 4;
  198. string source_data_node = 5;
  199. }
  200. message VolumeCopyResponse {
  201. uint64 last_append_at_ns = 1;
  202. }
  203. message CopyFileRequest {
  204. uint32 volume_id = 1;
  205. string ext = 2;
  206. uint32 compaction_revision = 3;
  207. uint64 stop_offset = 4;
  208. string collection = 5;
  209. bool is_ec_volume = 6;
  210. bool ignore_source_file_not_found = 7;
  211. }
  212. message CopyFileResponse {
  213. bytes file_content = 1;
  214. }
  215. message VolumeTailSenderRequest {
  216. uint32 volume_id = 1;
  217. uint64 since_ns = 2;
  218. uint32 idle_timeout_seconds = 3;
  219. }
  220. message VolumeTailSenderResponse {
  221. bytes needle_header = 1;
  222. bytes needle_body = 2;
  223. bool is_last_chunk = 3;
  224. }
  225. message VolumeTailReceiverRequest {
  226. uint32 volume_id = 1;
  227. uint64 since_ns = 2;
  228. uint32 idle_timeout_seconds = 3;
  229. string source_volume_server = 4;
  230. }
  231. message VolumeTailReceiverResponse {
  232. }
  233. message VolumeEcShardsGenerateRequest {
  234. uint32 volume_id = 1;
  235. string collection = 2;
  236. }
  237. message VolumeEcShardsGenerateResponse {
  238. }
  239. message VolumeEcShardsRebuildRequest {
  240. uint32 volume_id = 1;
  241. string collection = 2;
  242. }
  243. message VolumeEcShardsRebuildResponse {
  244. repeated uint32 rebuilt_shard_ids = 1;
  245. }
  246. message VolumeEcShardsCopyRequest {
  247. uint32 volume_id = 1;
  248. string collection = 2;
  249. repeated uint32 shard_ids = 3;
  250. bool copy_ecx_file = 4;
  251. string source_data_node = 5;
  252. bool copy_ecj_file = 6;
  253. bool copy_vif_file = 7;
  254. }
  255. message VolumeEcShardsCopyResponse {
  256. }
  257. message VolumeEcShardsDeleteRequest {
  258. uint32 volume_id = 1;
  259. string collection = 2;
  260. repeated uint32 shard_ids = 3;
  261. }
  262. message VolumeEcShardsDeleteResponse {
  263. }
  264. message VolumeEcShardsMountRequest {
  265. uint32 volume_id = 1;
  266. string collection = 2;
  267. repeated uint32 shard_ids = 3;
  268. }
  269. message VolumeEcShardsMountResponse {
  270. }
  271. message VolumeEcShardsUnmountRequest {
  272. uint32 volume_id = 1;
  273. repeated uint32 shard_ids = 3;
  274. }
  275. message VolumeEcShardsUnmountResponse {
  276. }
  277. message VolumeEcShardReadRequest {
  278. uint32 volume_id = 1;
  279. uint32 shard_id = 2;
  280. int64 offset = 3;
  281. int64 size = 4;
  282. uint64 file_key = 5;
  283. }
  284. message VolumeEcShardReadResponse {
  285. bytes data = 1;
  286. bool is_deleted = 2;
  287. }
  288. message VolumeEcBlobDeleteRequest {
  289. uint32 volume_id = 1;
  290. string collection = 2;
  291. uint64 file_key = 3;
  292. uint32 version = 4;
  293. }
  294. message VolumeEcBlobDeleteResponse {
  295. }
  296. message VolumeEcShardsToVolumeRequest {
  297. uint32 volume_id = 1;
  298. string collection = 2;
  299. }
  300. message VolumeEcShardsToVolumeResponse {
  301. }
  302. message ReadVolumeFileStatusRequest {
  303. uint32 volume_id = 1;
  304. }
  305. message ReadVolumeFileStatusResponse {
  306. uint32 volume_id = 1;
  307. uint64 idx_file_timestamp_seconds = 2;
  308. uint64 idx_file_size = 3;
  309. uint64 dat_file_timestamp_seconds = 4;
  310. uint64 dat_file_size = 5;
  311. uint64 file_count = 6;
  312. uint32 compaction_revision = 7;
  313. string collection = 8;
  314. }
  315. message DiskStatus {
  316. string dir = 1;
  317. uint64 all = 2;
  318. uint64 used = 3;
  319. uint64 free = 4;
  320. float percent_free = 5;
  321. float percent_used = 6;
  322. }
  323. message MemStatus {
  324. int32 goroutines = 1;
  325. uint64 all = 2;
  326. uint64 used = 3;
  327. uint64 free = 4;
  328. uint64 self = 5;
  329. uint64 heap = 6;
  330. uint64 stack = 7;
  331. }
  332. // tired storage on volume servers
  333. message RemoteFile {
  334. string backend_type = 1;
  335. string backend_id = 2;
  336. string key = 3;
  337. uint64 offset = 4;
  338. uint64 file_size = 5;
  339. uint64 modified_time = 6;
  340. string extension = 7;
  341. }
  342. message VolumeInfo {
  343. repeated RemoteFile files = 1;
  344. uint32 version = 2;
  345. string replication = 3;
  346. }
  347. message VolumeTierMoveDatToRemoteRequest {
  348. uint32 volume_id = 1;
  349. string collection = 2;
  350. string destination_backend_name = 3;
  351. bool keep_local_dat_file = 4;
  352. }
  353. message VolumeTierMoveDatToRemoteResponse {
  354. int64 processed = 1;
  355. float processedPercentage = 2;
  356. }
  357. message VolumeTierMoveDatFromRemoteRequest {
  358. uint32 volume_id = 1;
  359. string collection = 2;
  360. bool keep_remote_dat_file = 3;
  361. }
  362. message VolumeTierMoveDatFromRemoteResponse {
  363. int64 processed = 1;
  364. float processedPercentage = 2;
  365. }
  366. message VolumeServerStatusRequest {
  367. }
  368. message VolumeServerStatusResponse {
  369. repeated DiskStatus disk_statuses = 1;
  370. MemStatus memory_status = 2;
  371. }
  372. // select on volume servers
  373. message QueryRequest {
  374. repeated string selections = 1;
  375. repeated string from_file_ids = 2;
  376. message Filter {
  377. string field = 1;
  378. string operand = 2;
  379. string value = 3;
  380. }
  381. Filter filter = 3;
  382. message InputSerialization {
  383. // NONE | GZIP | BZIP2
  384. string compression_type = 1;
  385. message CSVInput {
  386. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  387. string record_delimiter = 2; // Default: \n
  388. string field_delimiter = 3; // Default: ,
  389. string quote_charactoer = 4; // Default: "
  390. string quote_escape_character = 5; // Default: "
  391. string comments = 6; // Default: #
  392. // If true, records might contain record delimiters within quote characters
  393. bool allow_quoted_record_delimiter = 7; // default False.
  394. }
  395. message JSONInput {
  396. string type = 1; // Valid values: DOCUMENT | LINES
  397. }
  398. message ParquetInput {
  399. }
  400. CSVInput csv_input = 2;
  401. JSONInput json_input = 3;
  402. ParquetInput parquet_input = 4;
  403. }
  404. InputSerialization input_serialization = 4;
  405. message OutputSerialization {
  406. message CSVOutput {
  407. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  408. string record_delimiter = 2; // Default: \n
  409. string field_delimiter = 3; // Default: ,
  410. string quote_charactoer = 4; // Default: "
  411. string quote_escape_character = 5; // Default: "
  412. }
  413. message JSONOutput {
  414. string record_delimiter = 1;
  415. }
  416. CSVOutput csv_output = 2;
  417. JSONOutput json_output = 3;
  418. }
  419. OutputSerialization output_serialization = 5;
  420. }
  421. message QueriedStripe {
  422. bytes records = 1;
  423. }