You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

624 lines
16 KiB

6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
3 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
4 months ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
  4. import "remote.proto";
  5. //////////////////////////////////////////////////
  6. service VolumeServer {
  7. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  8. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  9. }
  10. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  11. }
  12. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
  13. }
  14. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  15. }
  16. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  17. }
  18. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  19. }
  20. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  21. }
  22. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  23. }
  24. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  25. }
  26. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  27. }
  28. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  29. }
  30. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  31. }
  32. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  33. }
  34. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  35. }
  36. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  37. }
  38. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  39. }
  40. // copy the .idx .dat files, and mount this volume
  41. rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
  42. }
  43. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  44. }
  45. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  46. }
  47. rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
  48. }
  49. rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
  50. }
  51. rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
  52. }
  53. rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
  54. }
  55. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  56. }
  57. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  58. }
  59. // erasure coding
  60. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  61. }
  62. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  63. }
  64. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  65. }
  66. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  67. }
  68. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  69. }
  70. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  71. }
  72. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  73. }
  74. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  75. }
  76. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  77. }
  78. // tiered storage
  79. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  80. }
  81. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  82. }
  83. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  84. }
  85. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  86. }
  87. // remote storage
  88. rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
  89. }
  90. // <experimental> query
  91. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  92. }
  93. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  94. }
  95. rpc Ping (PingRequest) returns (PingResponse) {
  96. }
  97. }
  98. //////////////////////////////////////////////////
  99. message BatchDeleteRequest {
  100. repeated string file_ids = 1;
  101. bool skip_cookie_check = 2;
  102. }
  103. message BatchDeleteResponse {
  104. repeated DeleteResult results = 1;
  105. }
  106. message DeleteResult {
  107. string file_id = 1;
  108. int32 status = 2;
  109. string error = 3;
  110. uint32 size = 4;
  111. uint32 version = 5;
  112. }
  113. message Empty {
  114. }
  115. message VacuumVolumeCheckRequest {
  116. uint32 volume_id = 1;
  117. }
  118. message VacuumVolumeCheckResponse {
  119. double garbage_ratio = 1;
  120. }
  121. message VacuumVolumeCompactRequest {
  122. uint32 volume_id = 1;
  123. int64 preallocate = 2;
  124. }
  125. message VacuumVolumeCompactResponse {
  126. int64 processed_bytes = 1;
  127. float load_avg_1m = 2;
  128. }
  129. message VacuumVolumeCommitRequest {
  130. uint32 volume_id = 1;
  131. }
  132. message VacuumVolumeCommitResponse {
  133. bool is_read_only = 1;
  134. uint64 volume_size = 2;
  135. }
  136. message VacuumVolumeCleanupRequest {
  137. uint32 volume_id = 1;
  138. }
  139. message VacuumVolumeCleanupResponse {
  140. }
  141. message DeleteCollectionRequest {
  142. string collection = 1;
  143. }
  144. message DeleteCollectionResponse {
  145. }
  146. message AllocateVolumeRequest {
  147. uint32 volume_id = 1;
  148. string collection = 2;
  149. int64 preallocate = 3;
  150. string replication = 4;
  151. string ttl = 5;
  152. uint32 memory_map_max_size_mb = 6;
  153. string disk_type = 7;
  154. }
  155. message AllocateVolumeResponse {
  156. }
  157. message VolumeSyncStatusRequest {
  158. uint32 volume_id = 1;
  159. }
  160. message VolumeSyncStatusResponse {
  161. uint32 volume_id = 1;
  162. string collection = 2;
  163. string replication = 4;
  164. string ttl = 5;
  165. uint64 tail_offset = 6;
  166. uint32 compact_revision = 7;
  167. uint64 idx_file_size = 8;
  168. }
  169. message VolumeIncrementalCopyRequest {
  170. uint32 volume_id = 1;
  171. uint64 since_ns = 2;
  172. }
  173. message VolumeIncrementalCopyResponse {
  174. bytes file_content = 1;
  175. }
  176. message VolumeMountRequest {
  177. uint32 volume_id = 1;
  178. }
  179. message VolumeMountResponse {
  180. }
  181. message VolumeUnmountRequest {
  182. uint32 volume_id = 1;
  183. }
  184. message VolumeUnmountResponse {
  185. }
  186. message VolumeDeleteRequest {
  187. uint32 volume_id = 1;
  188. bool only_empty = 2;
  189. }
  190. message VolumeDeleteResponse {
  191. }
  192. message VolumeMarkReadonlyRequest {
  193. uint32 volume_id = 1;
  194. }
  195. message VolumeMarkReadonlyResponse {
  196. }
  197. message VolumeMarkWritableRequest {
  198. uint32 volume_id = 1;
  199. }
  200. message VolumeMarkWritableResponse {
  201. }
  202. message VolumeConfigureRequest {
  203. uint32 volume_id = 1;
  204. string replication = 2;
  205. }
  206. message VolumeConfigureResponse {
  207. string error = 1;
  208. }
  209. message VolumeStatusRequest {
  210. uint32 volume_id = 1;
  211. }
  212. message VolumeStatusResponse {
  213. bool is_read_only = 1;
  214. uint64 volume_size = 2;
  215. uint64 file_count = 3;
  216. uint64 file_deleted_count = 4;
  217. }
  218. message VolumeCopyRequest {
  219. uint32 volume_id = 1;
  220. string collection = 2;
  221. string replication = 3;
  222. string ttl = 4;
  223. string source_data_node = 5;
  224. string disk_type = 6;
  225. int64 io_byte_per_second = 7;
  226. }
  227. message VolumeCopyResponse {
  228. uint64 last_append_at_ns = 1;
  229. int64 processed_bytes = 2;
  230. }
  231. message CopyFileRequest {
  232. uint32 volume_id = 1;
  233. string ext = 2;
  234. uint32 compaction_revision = 3;
  235. uint64 stop_offset = 4;
  236. string collection = 5;
  237. bool is_ec_volume = 6;
  238. bool ignore_source_file_not_found = 7;
  239. }
  240. message CopyFileResponse {
  241. bytes file_content = 1;
  242. int64 modified_ts_ns = 2;
  243. }
  244. message ReadNeedleBlobRequest {
  245. uint32 volume_id = 1;
  246. int64 offset = 3; // actual offset
  247. int32 size = 4;
  248. }
  249. message ReadNeedleBlobResponse {
  250. bytes needle_blob = 1;
  251. }
  252. message ReadNeedleMetaRequest {
  253. uint32 volume_id = 1;
  254. uint64 needle_id = 2;
  255. int64 offset = 3; // actual offset
  256. int32 size = 4;
  257. }
  258. message ReadNeedleMetaResponse {
  259. uint32 cookie = 1;
  260. uint64 last_modified = 2;
  261. uint32 crc = 3;
  262. string ttl = 4;
  263. uint64 append_at_ns = 5;
  264. }
  265. message WriteNeedleBlobRequest {
  266. uint32 volume_id = 1;
  267. uint64 needle_id = 2;
  268. int32 size = 3;
  269. bytes needle_blob = 4;
  270. }
  271. message WriteNeedleBlobResponse {
  272. }
  273. message ReadAllNeedlesRequest {
  274. repeated uint32 volume_ids = 1;
  275. }
  276. message ReadAllNeedlesResponse {
  277. uint32 volume_id = 1;
  278. uint64 needle_id = 2;
  279. uint32 cookie = 3;
  280. bytes needle_blob = 5;
  281. bool needle_blob_compressed = 6;
  282. uint64 last_modified = 7;
  283. uint32 crc = 8;
  284. bytes name = 9;
  285. bytes mime = 10;
  286. }
  287. message VolumeTailSenderRequest {
  288. uint32 volume_id = 1;
  289. uint64 since_ns = 2;
  290. uint32 idle_timeout_seconds = 3;
  291. }
  292. message VolumeTailSenderResponse {
  293. bytes needle_header = 1;
  294. bytes needle_body = 2;
  295. bool is_last_chunk = 3;
  296. }
  297. message VolumeTailReceiverRequest {
  298. uint32 volume_id = 1;
  299. uint64 since_ns = 2;
  300. uint32 idle_timeout_seconds = 3;
  301. string source_volume_server = 4;
  302. }
  303. message VolumeTailReceiverResponse {
  304. }
  305. message VolumeEcShardsGenerateRequest {
  306. uint32 volume_id = 1;
  307. string collection = 2;
  308. }
  309. message VolumeEcShardsGenerateResponse {
  310. }
  311. message VolumeEcShardsRebuildRequest {
  312. uint32 volume_id = 1;
  313. string collection = 2;
  314. }
  315. message VolumeEcShardsRebuildResponse {
  316. repeated uint32 rebuilt_shard_ids = 1;
  317. }
  318. message VolumeEcShardsCopyRequest {
  319. uint32 volume_id = 1;
  320. string collection = 2;
  321. repeated uint32 shard_ids = 3;
  322. bool copy_ecx_file = 4;
  323. string source_data_node = 5;
  324. bool copy_ecj_file = 6;
  325. bool copy_vif_file = 7;
  326. }
  327. message VolumeEcShardsCopyResponse {
  328. }
  329. message VolumeEcShardsDeleteRequest {
  330. uint32 volume_id = 1;
  331. string collection = 2;
  332. repeated uint32 shard_ids = 3;
  333. }
  334. message VolumeEcShardsDeleteResponse {
  335. }
  336. message VolumeEcShardsMountRequest {
  337. uint32 volume_id = 1;
  338. string collection = 2;
  339. repeated uint32 shard_ids = 3;
  340. }
  341. message VolumeEcShardsMountResponse {
  342. }
  343. message VolumeEcShardsUnmountRequest {
  344. uint32 volume_id = 1;
  345. repeated uint32 shard_ids = 3;
  346. }
  347. message VolumeEcShardsUnmountResponse {
  348. }
  349. message VolumeEcShardReadRequest {
  350. uint32 volume_id = 1;
  351. uint32 shard_id = 2;
  352. int64 offset = 3;
  353. int64 size = 4;
  354. uint64 file_key = 5;
  355. }
  356. message VolumeEcShardReadResponse {
  357. bytes data = 1;
  358. bool is_deleted = 2;
  359. }
  360. message VolumeEcBlobDeleteRequest {
  361. uint32 volume_id = 1;
  362. string collection = 2;
  363. uint64 file_key = 3;
  364. uint32 version = 4;
  365. }
  366. message VolumeEcBlobDeleteResponse {
  367. }
  368. message VolumeEcShardsToVolumeRequest {
  369. uint32 volume_id = 1;
  370. string collection = 2;
  371. }
  372. message VolumeEcShardsToVolumeResponse {
  373. }
  374. message ReadVolumeFileStatusRequest {
  375. uint32 volume_id = 1;
  376. }
  377. message ReadVolumeFileStatusResponse {
  378. uint32 volume_id = 1;
  379. uint64 idx_file_timestamp_seconds = 2;
  380. uint64 idx_file_size = 3;
  381. uint64 dat_file_timestamp_seconds = 4;
  382. uint64 dat_file_size = 5;
  383. uint64 file_count = 6;
  384. uint32 compaction_revision = 7;
  385. string collection = 8;
  386. string disk_type = 9;
  387. VolumeInfo volume_info = 10;
  388. }
  389. message DiskStatus {
  390. string dir = 1;
  391. uint64 all = 2;
  392. uint64 used = 3;
  393. uint64 free = 4;
  394. float percent_free = 5;
  395. float percent_used = 6;
  396. string disk_type = 7;
  397. }
  398. message MemStatus {
  399. int32 goroutines = 1;
  400. uint64 all = 2;
  401. uint64 used = 3;
  402. uint64 free = 4;
  403. uint64 self = 5;
  404. uint64 heap = 6;
  405. uint64 stack = 7;
  406. }
  407. // tired storage on volume servers
  408. message RemoteFile {
  409. string backend_type = 1;
  410. string backend_id = 2;
  411. string key = 3;
  412. uint64 offset = 4;
  413. uint64 file_size = 5;
  414. uint64 modified_time = 6;
  415. string extension = 7;
  416. }
  417. message VolumeInfo {
  418. repeated RemoteFile files = 1;
  419. uint32 version = 2;
  420. string replication = 3;
  421. uint32 BytesOffset = 4;
  422. int64 dat_file_size = 5; // used for EC encoded volumes to store the original file size
  423. uint64 DestroyTime = 6; // used to record the destruction time of ec volume
  424. }
  425. // tiered storage
  426. message VolumeTierMoveDatToRemoteRequest {
  427. uint32 volume_id = 1;
  428. string collection = 2;
  429. string destination_backend_name = 3;
  430. bool keep_local_dat_file = 4;
  431. }
  432. message VolumeTierMoveDatToRemoteResponse {
  433. int64 processed = 1;
  434. float processedPercentage = 2;
  435. }
  436. message VolumeTierMoveDatFromRemoteRequest {
  437. uint32 volume_id = 1;
  438. string collection = 2;
  439. bool keep_remote_dat_file = 3;
  440. }
  441. message VolumeTierMoveDatFromRemoteResponse {
  442. int64 processed = 1;
  443. float processedPercentage = 2;
  444. }
  445. message VolumeServerStatusRequest {
  446. }
  447. message VolumeServerStatusResponse {
  448. repeated DiskStatus disk_statuses = 1;
  449. MemStatus memory_status = 2;
  450. string version = 3;
  451. string data_center = 4;
  452. string rack = 5;
  453. }
  454. message VolumeServerLeaveRequest {
  455. }
  456. message VolumeServerLeaveResponse {
  457. }
  458. // remote storage
  459. message FetchAndWriteNeedleRequest {
  460. uint32 volume_id = 1;
  461. uint64 needle_id = 2;
  462. uint32 cookie = 3;
  463. int64 offset = 4;
  464. int64 size = 5;
  465. message Replica {
  466. string url = 1;
  467. string public_url = 2;
  468. int32 grpc_port = 3;
  469. }
  470. repeated Replica replicas = 6;
  471. string auth = 7;
  472. // remote conf
  473. remote_pb.RemoteConf remote_conf = 15;
  474. remote_pb.RemoteStorageLocation remote_location = 16;
  475. }
  476. message FetchAndWriteNeedleResponse {
  477. string e_tag = 1;
  478. }
  479. // select on volume servers
  480. message QueryRequest {
  481. repeated string selections = 1;
  482. repeated string from_file_ids = 2;
  483. message Filter {
  484. string field = 1;
  485. string operand = 2;
  486. string value = 3;
  487. }
  488. Filter filter = 3;
  489. message InputSerialization {
  490. // NONE | GZIP | BZIP2
  491. string compression_type = 1;
  492. message CSVInput {
  493. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  494. string record_delimiter = 2; // Default: \n
  495. string field_delimiter = 3; // Default: ,
  496. string quote_character = 4; // Default: "
  497. string quote_escape_character = 5; // Default: "
  498. string comments = 6; // Default: #
  499. // If true, records might contain record delimiters within quote characters
  500. bool allow_quoted_record_delimiter = 7; // default False.
  501. }
  502. message JSONInput {
  503. string type = 1; // Valid values: DOCUMENT | LINES
  504. }
  505. message ParquetInput {
  506. }
  507. CSVInput csv_input = 2;
  508. JSONInput json_input = 3;
  509. ParquetInput parquet_input = 4;
  510. }
  511. InputSerialization input_serialization = 4;
  512. message OutputSerialization {
  513. message CSVOutput {
  514. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  515. string record_delimiter = 2; // Default: \n
  516. string field_delimiter = 3; // Default: ,
  517. string quote_character = 4; // Default: "
  518. string quote_escape_character = 5; // Default: "
  519. }
  520. message JSONOutput {
  521. string record_delimiter = 1;
  522. }
  523. CSVOutput csv_output = 2;
  524. JSONOutput json_output = 3;
  525. }
  526. OutputSerialization output_serialization = 5;
  527. }
  528. message QueriedStripe {
  529. bytes records = 1;
  530. }
  531. message VolumeNeedleStatusRequest {
  532. uint32 volume_id = 1;
  533. uint64 needle_id = 2;
  534. }
  535. message VolumeNeedleStatusResponse {
  536. uint64 needle_id = 1;
  537. uint32 cookie = 2;
  538. uint32 size = 3;
  539. uint64 last_modified = 4;
  540. uint32 crc = 5;
  541. string ttl = 6;
  542. }
  543. message PingRequest {
  544. string target = 1; // default to ping itself
  545. string target_type = 2;
  546. }
  547. message PingResponse {
  548. int64 start_time_ns = 1;
  549. int64 remote_time_ns = 2;
  550. int64 stop_time_ns = 3;
  551. }