You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

617 lines
15 KiB

6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
3 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
  4. import "remote.proto";
  5. //////////////////////////////////////////////////
  6. service VolumeServer {
  7. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  8. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  9. }
  10. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  11. }
  12. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
  13. }
  14. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  15. }
  16. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  17. }
  18. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  19. }
  20. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  21. }
  22. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  23. }
  24. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  25. }
  26. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  27. }
  28. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  29. }
  30. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  31. }
  32. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  33. }
  34. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  35. }
  36. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  37. }
  38. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  39. }
  40. // copy the .idx .dat files, and mount this volume
  41. rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
  42. }
  43. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  44. }
  45. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  46. }
  47. rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
  48. }
  49. rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
  50. }
  51. rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
  52. }
  53. rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
  54. }
  55. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  56. }
  57. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  58. }
  59. // erasure coding
  60. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  61. }
  62. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  63. }
  64. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  65. }
  66. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  67. }
  68. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  69. }
  70. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  71. }
  72. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  73. }
  74. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  75. }
  76. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  77. }
  78. // tiered storage
  79. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  80. }
  81. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  82. }
  83. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  84. }
  85. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  86. }
  87. // remote storage
  88. rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
  89. }
  90. // <experimental> query
  91. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  92. }
  93. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  94. }
  95. rpc Ping (PingRequest) returns (PingResponse) {
  96. }
  97. }
  98. //////////////////////////////////////////////////
  99. message BatchDeleteRequest {
  100. repeated string file_ids = 1;
  101. bool skip_cookie_check = 2;
  102. }
  103. message BatchDeleteResponse {
  104. repeated DeleteResult results = 1;
  105. }
  106. message DeleteResult {
  107. string file_id = 1;
  108. int32 status = 2;
  109. string error = 3;
  110. uint32 size = 4;
  111. uint32 version = 5;
  112. }
  113. message Empty {
  114. }
  115. message VacuumVolumeCheckRequest {
  116. uint32 volume_id = 1;
  117. }
  118. message VacuumVolumeCheckResponse {
  119. double garbage_ratio = 1;
  120. }
  121. message VacuumVolumeCompactRequest {
  122. uint32 volume_id = 1;
  123. int64 preallocate = 2;
  124. }
  125. message VacuumVolumeCompactResponse {
  126. int64 processed_bytes = 1;
  127. float load_avg_1m = 2;
  128. }
  129. message VacuumVolumeCommitRequest {
  130. uint32 volume_id = 1;
  131. }
  132. message VacuumVolumeCommitResponse {
  133. bool is_read_only = 1;
  134. }
  135. message VacuumVolumeCleanupRequest {
  136. uint32 volume_id = 1;
  137. }
  138. message VacuumVolumeCleanupResponse {
  139. }
  140. message DeleteCollectionRequest {
  141. string collection = 1;
  142. }
  143. message DeleteCollectionResponse {
  144. }
  145. message AllocateVolumeRequest {
  146. uint32 volume_id = 1;
  147. string collection = 2;
  148. int64 preallocate = 3;
  149. string replication = 4;
  150. string ttl = 5;
  151. uint32 memory_map_max_size_mb = 6;
  152. string disk_type = 7;
  153. }
  154. message AllocateVolumeResponse {
  155. }
  156. message VolumeSyncStatusRequest {
  157. uint32 volume_id = 1;
  158. }
  159. message VolumeSyncStatusResponse {
  160. uint32 volume_id = 1;
  161. string collection = 2;
  162. string replication = 4;
  163. string ttl = 5;
  164. uint64 tail_offset = 6;
  165. uint32 compact_revision = 7;
  166. uint64 idx_file_size = 8;
  167. }
  168. message VolumeIncrementalCopyRequest {
  169. uint32 volume_id = 1;
  170. uint64 since_ns = 2;
  171. }
  172. message VolumeIncrementalCopyResponse {
  173. bytes file_content = 1;
  174. }
  175. message VolumeMountRequest {
  176. uint32 volume_id = 1;
  177. }
  178. message VolumeMountResponse {
  179. }
  180. message VolumeUnmountRequest {
  181. uint32 volume_id = 1;
  182. }
  183. message VolumeUnmountResponse {
  184. }
  185. message VolumeDeleteRequest {
  186. uint32 volume_id = 1;
  187. }
  188. message VolumeDeleteResponse {
  189. }
  190. message VolumeMarkReadonlyRequest {
  191. uint32 volume_id = 1;
  192. }
  193. message VolumeMarkReadonlyResponse {
  194. }
  195. message VolumeMarkWritableRequest {
  196. uint32 volume_id = 1;
  197. }
  198. message VolumeMarkWritableResponse {
  199. }
  200. message VolumeConfigureRequest {
  201. uint32 volume_id = 1;
  202. string replication = 2;
  203. }
  204. message VolumeConfigureResponse {
  205. string error = 1;
  206. }
  207. message VolumeStatusRequest {
  208. uint32 volume_id = 1;
  209. }
  210. message VolumeStatusResponse {
  211. bool is_read_only = 1;
  212. }
  213. message VolumeCopyRequest {
  214. uint32 volume_id = 1;
  215. string collection = 2;
  216. string replication = 3;
  217. string ttl = 4;
  218. string source_data_node = 5;
  219. string disk_type = 6;
  220. int64 io_byte_per_second = 7;
  221. }
  222. message VolumeCopyResponse {
  223. uint64 last_append_at_ns = 1;
  224. int64 processed_bytes = 2;
  225. }
  226. message CopyFileRequest {
  227. uint32 volume_id = 1;
  228. string ext = 2;
  229. uint32 compaction_revision = 3;
  230. uint64 stop_offset = 4;
  231. string collection = 5;
  232. bool is_ec_volume = 6;
  233. bool ignore_source_file_not_found = 7;
  234. }
  235. message CopyFileResponse {
  236. bytes file_content = 1;
  237. int64 modified_ts_ns = 2;
  238. }
  239. message ReadNeedleBlobRequest {
  240. uint32 volume_id = 1;
  241. int64 offset = 3; // actual offset
  242. int32 size = 4;
  243. }
  244. message ReadNeedleBlobResponse {
  245. bytes needle_blob = 1;
  246. }
  247. message ReadNeedleMetaRequest {
  248. uint32 volume_id = 1;
  249. uint64 needle_id = 2;
  250. int64 offset = 3; // actual offset
  251. int32 size = 4;
  252. }
  253. message ReadNeedleMetaResponse {
  254. uint32 cookie = 1;
  255. uint64 last_modified = 2;
  256. uint32 crc = 3;
  257. string ttl = 4;
  258. uint64 append_at_ns = 5;
  259. }
  260. message WriteNeedleBlobRequest {
  261. uint32 volume_id = 1;
  262. uint64 needle_id = 2;
  263. int32 size = 3;
  264. bytes needle_blob = 4;
  265. }
  266. message WriteNeedleBlobResponse {
  267. }
  268. message ReadAllNeedlesRequest {
  269. repeated uint32 volume_ids = 1;
  270. }
  271. message ReadAllNeedlesResponse {
  272. uint32 volume_id = 1;
  273. uint64 needle_id = 2;
  274. uint32 cookie = 3;
  275. bytes needle_blob = 5;
  276. bool needle_blob_compressed = 6;
  277. uint64 last_modified = 7;
  278. uint32 crc = 8;
  279. bytes name = 9;
  280. bytes mime = 10;
  281. }
  282. message VolumeTailSenderRequest {
  283. uint32 volume_id = 1;
  284. uint64 since_ns = 2;
  285. uint32 idle_timeout_seconds = 3;
  286. }
  287. message VolumeTailSenderResponse {
  288. bytes needle_header = 1;
  289. bytes needle_body = 2;
  290. bool is_last_chunk = 3;
  291. }
  292. message VolumeTailReceiverRequest {
  293. uint32 volume_id = 1;
  294. uint64 since_ns = 2;
  295. uint32 idle_timeout_seconds = 3;
  296. string source_volume_server = 4;
  297. }
  298. message VolumeTailReceiverResponse {
  299. }
  300. message VolumeEcShardsGenerateRequest {
  301. uint32 volume_id = 1;
  302. string collection = 2;
  303. }
  304. message VolumeEcShardsGenerateResponse {
  305. }
  306. message VolumeEcShardsRebuildRequest {
  307. uint32 volume_id = 1;
  308. string collection = 2;
  309. }
  310. message VolumeEcShardsRebuildResponse {
  311. repeated uint32 rebuilt_shard_ids = 1;
  312. }
  313. message VolumeEcShardsCopyRequest {
  314. uint32 volume_id = 1;
  315. string collection = 2;
  316. repeated uint32 shard_ids = 3;
  317. bool copy_ecx_file = 4;
  318. string source_data_node = 5;
  319. bool copy_ecj_file = 6;
  320. bool copy_vif_file = 7;
  321. }
  322. message VolumeEcShardsCopyResponse {
  323. }
  324. message VolumeEcShardsDeleteRequest {
  325. uint32 volume_id = 1;
  326. string collection = 2;
  327. repeated uint32 shard_ids = 3;
  328. }
  329. message VolumeEcShardsDeleteResponse {
  330. }
  331. message VolumeEcShardsMountRequest {
  332. uint32 volume_id = 1;
  333. string collection = 2;
  334. repeated uint32 shard_ids = 3;
  335. }
  336. message VolumeEcShardsMountResponse {
  337. }
  338. message VolumeEcShardsUnmountRequest {
  339. uint32 volume_id = 1;
  340. repeated uint32 shard_ids = 3;
  341. }
  342. message VolumeEcShardsUnmountResponse {
  343. }
  344. message VolumeEcShardReadRequest {
  345. uint32 volume_id = 1;
  346. uint32 shard_id = 2;
  347. int64 offset = 3;
  348. int64 size = 4;
  349. uint64 file_key = 5;
  350. }
  351. message VolumeEcShardReadResponse {
  352. bytes data = 1;
  353. bool is_deleted = 2;
  354. }
  355. message VolumeEcBlobDeleteRequest {
  356. uint32 volume_id = 1;
  357. string collection = 2;
  358. uint64 file_key = 3;
  359. uint32 version = 4;
  360. }
  361. message VolumeEcBlobDeleteResponse {
  362. }
  363. message VolumeEcShardsToVolumeRequest {
  364. uint32 volume_id = 1;
  365. string collection = 2;
  366. }
  367. message VolumeEcShardsToVolumeResponse {
  368. }
  369. message ReadVolumeFileStatusRequest {
  370. uint32 volume_id = 1;
  371. }
  372. message ReadVolumeFileStatusResponse {
  373. uint32 volume_id = 1;
  374. uint64 idx_file_timestamp_seconds = 2;
  375. uint64 idx_file_size = 3;
  376. uint64 dat_file_timestamp_seconds = 4;
  377. uint64 dat_file_size = 5;
  378. uint64 file_count = 6;
  379. uint32 compaction_revision = 7;
  380. string collection = 8;
  381. string disk_type = 9;
  382. VolumeInfo volume_info = 10;
  383. }
  384. message DiskStatus {
  385. string dir = 1;
  386. uint64 all = 2;
  387. uint64 used = 3;
  388. uint64 free = 4;
  389. float percent_free = 5;
  390. float percent_used = 6;
  391. string disk_type = 7;
  392. }
  393. message MemStatus {
  394. int32 goroutines = 1;
  395. uint64 all = 2;
  396. uint64 used = 3;
  397. uint64 free = 4;
  398. uint64 self = 5;
  399. uint64 heap = 6;
  400. uint64 stack = 7;
  401. }
  402. // tired storage on volume servers
  403. message RemoteFile {
  404. string backend_type = 1;
  405. string backend_id = 2;
  406. string key = 3;
  407. uint64 offset = 4;
  408. uint64 file_size = 5;
  409. uint64 modified_time = 6;
  410. string extension = 7;
  411. }
  412. message VolumeInfo {
  413. repeated RemoteFile files = 1;
  414. uint32 version = 2;
  415. string replication = 3;
  416. uint32 BytesOffset = 4;
  417. }
  418. // tiered storage
  419. message VolumeTierMoveDatToRemoteRequest {
  420. uint32 volume_id = 1;
  421. string collection = 2;
  422. string destination_backend_name = 3;
  423. bool keep_local_dat_file = 4;
  424. }
  425. message VolumeTierMoveDatToRemoteResponse {
  426. int64 processed = 1;
  427. float processedPercentage = 2;
  428. }
  429. message VolumeTierMoveDatFromRemoteRequest {
  430. uint32 volume_id = 1;
  431. string collection = 2;
  432. bool keep_remote_dat_file = 3;
  433. }
  434. message VolumeTierMoveDatFromRemoteResponse {
  435. int64 processed = 1;
  436. float processedPercentage = 2;
  437. }
  438. message VolumeServerStatusRequest {
  439. }
  440. message VolumeServerStatusResponse {
  441. repeated DiskStatus disk_statuses = 1;
  442. MemStatus memory_status = 2;
  443. string version = 3;
  444. string data_center = 4;
  445. string rack = 5;
  446. }
  447. message VolumeServerLeaveRequest {
  448. }
  449. message VolumeServerLeaveResponse {
  450. }
  451. // remote storage
  452. message FetchAndWriteNeedleRequest {
  453. uint32 volume_id = 1;
  454. uint64 needle_id = 2;
  455. uint32 cookie = 3;
  456. int64 offset = 4;
  457. int64 size = 5;
  458. message Replica {
  459. string url = 1;
  460. string public_url = 2;
  461. int32 grpc_port = 3;
  462. }
  463. repeated Replica replicas = 6;
  464. string auth = 7;
  465. // remote conf
  466. remote_pb.RemoteConf remote_conf = 15;
  467. remote_pb.RemoteStorageLocation remote_location = 16;
  468. }
  469. message FetchAndWriteNeedleResponse {
  470. string e_tag = 1;
  471. }
  472. // select on volume servers
  473. message QueryRequest {
  474. repeated string selections = 1;
  475. repeated string from_file_ids = 2;
  476. message Filter {
  477. string field = 1;
  478. string operand = 2;
  479. string value = 3;
  480. }
  481. Filter filter = 3;
  482. message InputSerialization {
  483. // NONE | GZIP | BZIP2
  484. string compression_type = 1;
  485. message CSVInput {
  486. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  487. string record_delimiter = 2; // Default: \n
  488. string field_delimiter = 3; // Default: ,
  489. string quote_character = 4; // Default: "
  490. string quote_escape_character = 5; // Default: "
  491. string comments = 6; // Default: #
  492. // If true, records might contain record delimiters within quote characters
  493. bool allow_quoted_record_delimiter = 7; // default False.
  494. }
  495. message JSONInput {
  496. string type = 1; // Valid values: DOCUMENT | LINES
  497. }
  498. message ParquetInput {
  499. }
  500. CSVInput csv_input = 2;
  501. JSONInput json_input = 3;
  502. ParquetInput parquet_input = 4;
  503. }
  504. InputSerialization input_serialization = 4;
  505. message OutputSerialization {
  506. message CSVOutput {
  507. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  508. string record_delimiter = 2; // Default: \n
  509. string field_delimiter = 3; // Default: ,
  510. string quote_character = 4; // Default: "
  511. string quote_escape_character = 5; // Default: "
  512. }
  513. message JSONOutput {
  514. string record_delimiter = 1;
  515. }
  516. CSVOutput csv_output = 2;
  517. JSONOutput json_output = 3;
  518. }
  519. OutputSerialization output_serialization = 5;
  520. }
  521. message QueriedStripe {
  522. bytes records = 1;
  523. }
  524. message VolumeNeedleStatusRequest {
  525. uint32 volume_id = 1;
  526. uint64 needle_id = 2;
  527. }
  528. message VolumeNeedleStatusResponse {
  529. uint64 needle_id = 1;
  530. uint32 cookie = 2;
  531. uint32 size = 3;
  532. uint64 last_modified = 4;
  533. uint32 crc = 5;
  534. string ttl = 6;
  535. }
  536. message PingRequest {
  537. string target = 1; // default to ping itself
  538. string target_type = 2;
  539. }
  540. message PingResponse {
  541. int64 start_time_ns = 1;
  542. int64 remote_time_ns = 2;
  543. int64 stop_time_ns = 3;
  544. }