You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

594 lines
15 KiB

6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
3 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
  4. import "remote.proto";
  5. //////////////////////////////////////////////////
  6. service VolumeServer {
  7. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  8. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  9. }
  10. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  11. }
  12. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
  13. }
  14. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  15. }
  16. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  17. }
  18. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  19. }
  20. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  21. }
  22. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  23. }
  24. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  25. }
  26. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  27. }
  28. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  29. }
  30. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  31. }
  32. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  33. }
  34. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  35. }
  36. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  37. }
  38. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  39. }
  40. // copy the .idx .dat files, and mount this volume
  41. rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
  42. }
  43. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  44. }
  45. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  46. }
  47. rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
  48. }
  49. rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
  50. }
  51. rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
  52. }
  53. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  54. }
  55. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  56. }
  57. // erasure coding
  58. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  59. }
  60. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  61. }
  62. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  63. }
  64. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  65. }
  66. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  67. }
  68. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  69. }
  70. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  71. }
  72. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  73. }
  74. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  75. }
  76. // tiered storage
  77. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  78. }
  79. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  80. }
  81. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  82. }
  83. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  84. }
  85. // remote storage
  86. rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
  87. }
  88. // <experimental> query
  89. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  90. }
  91. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  92. }
  93. rpc Ping (PingRequest) returns (PingResponse) {
  94. }
  95. }
  96. //////////////////////////////////////////////////
  97. message BatchDeleteRequest {
  98. repeated string file_ids = 1;
  99. bool skip_cookie_check = 2;
  100. }
  101. message BatchDeleteResponse {
  102. repeated DeleteResult results = 1;
  103. }
  104. message DeleteResult {
  105. string file_id = 1;
  106. int32 status = 2;
  107. string error = 3;
  108. uint32 size = 4;
  109. uint32 version = 5;
  110. }
  111. message Empty {
  112. }
  113. message VacuumVolumeCheckRequest {
  114. uint32 volume_id = 1;
  115. }
  116. message VacuumVolumeCheckResponse {
  117. double garbage_ratio = 1;
  118. }
  119. message VacuumVolumeCompactRequest {
  120. uint32 volume_id = 1;
  121. int64 preallocate = 2;
  122. }
  123. message VacuumVolumeCompactResponse {
  124. int64 processed_bytes = 1;
  125. float load_avg_1m = 2;
  126. }
  127. message VacuumVolumeCommitRequest {
  128. uint32 volume_id = 1;
  129. }
  130. message VacuumVolumeCommitResponse {
  131. bool is_read_only = 1;
  132. }
  133. message VacuumVolumeCleanupRequest {
  134. uint32 volume_id = 1;
  135. }
  136. message VacuumVolumeCleanupResponse {
  137. }
  138. message DeleteCollectionRequest {
  139. string collection = 1;
  140. }
  141. message DeleteCollectionResponse {
  142. }
  143. message AllocateVolumeRequest {
  144. uint32 volume_id = 1;
  145. string collection = 2;
  146. int64 preallocate = 3;
  147. string replication = 4;
  148. string ttl = 5;
  149. uint32 memory_map_max_size_mb = 6;
  150. string disk_type = 7;
  151. }
  152. message AllocateVolumeResponse {
  153. }
  154. message VolumeSyncStatusRequest {
  155. uint32 volume_id = 1;
  156. }
  157. message VolumeSyncStatusResponse {
  158. uint32 volume_id = 1;
  159. string collection = 2;
  160. string replication = 4;
  161. string ttl = 5;
  162. uint64 tail_offset = 6;
  163. uint32 compact_revision = 7;
  164. uint64 idx_file_size = 8;
  165. }
  166. message VolumeIncrementalCopyRequest {
  167. uint32 volume_id = 1;
  168. uint64 since_ns = 2;
  169. }
  170. message VolumeIncrementalCopyResponse {
  171. bytes file_content = 1;
  172. }
  173. message VolumeMountRequest {
  174. uint32 volume_id = 1;
  175. }
  176. message VolumeMountResponse {
  177. }
  178. message VolumeUnmountRequest {
  179. uint32 volume_id = 1;
  180. }
  181. message VolumeUnmountResponse {
  182. }
  183. message VolumeDeleteRequest {
  184. uint32 volume_id = 1;
  185. }
  186. message VolumeDeleteResponse {
  187. }
  188. message VolumeMarkReadonlyRequest {
  189. uint32 volume_id = 1;
  190. }
  191. message VolumeMarkReadonlyResponse {
  192. }
  193. message VolumeMarkWritableRequest {
  194. uint32 volume_id = 1;
  195. }
  196. message VolumeMarkWritableResponse {
  197. }
  198. message VolumeConfigureRequest {
  199. uint32 volume_id = 1;
  200. string replication = 2;
  201. }
  202. message VolumeConfigureResponse {
  203. string error = 1;
  204. }
  205. message VolumeStatusRequest {
  206. uint32 volume_id = 1;
  207. }
  208. message VolumeStatusResponse {
  209. bool is_read_only = 1;
  210. }
  211. message VolumeCopyRequest {
  212. uint32 volume_id = 1;
  213. string collection = 2;
  214. string replication = 3;
  215. string ttl = 4;
  216. string source_data_node = 5;
  217. string disk_type = 6;
  218. int64 io_byte_per_second = 7;
  219. }
  220. message VolumeCopyResponse {
  221. uint64 last_append_at_ns = 1;
  222. int64 processed_bytes = 2;
  223. }
  224. message CopyFileRequest {
  225. uint32 volume_id = 1;
  226. string ext = 2;
  227. uint32 compaction_revision = 3;
  228. uint64 stop_offset = 4;
  229. string collection = 5;
  230. bool is_ec_volume = 6;
  231. bool ignore_source_file_not_found = 7;
  232. }
  233. message CopyFileResponse {
  234. bytes file_content = 1;
  235. int64 modified_ts_ns = 2;
  236. }
  237. message ReadNeedleBlobRequest {
  238. uint32 volume_id = 1;
  239. uint64 needle_id = 2;
  240. int64 offset = 3; // actual offset
  241. int32 size = 4;
  242. }
  243. message ReadNeedleBlobResponse {
  244. bytes needle_blob = 1;
  245. }
  246. message WriteNeedleBlobRequest {
  247. uint32 volume_id = 1;
  248. uint64 needle_id = 2;
  249. int32 size = 3;
  250. bytes needle_blob = 4;
  251. }
  252. message WriteNeedleBlobResponse {
  253. }
  254. message ReadAllNeedlesRequest {
  255. repeated uint32 volume_ids = 1;
  256. }
  257. message ReadAllNeedlesResponse {
  258. uint32 volume_id = 1;
  259. uint64 needle_id = 2;
  260. uint32 cookie = 3;
  261. bytes needle_blob = 5;
  262. }
  263. message VolumeTailSenderRequest {
  264. uint32 volume_id = 1;
  265. uint64 since_ns = 2;
  266. uint32 idle_timeout_seconds = 3;
  267. }
  268. message VolumeTailSenderResponse {
  269. bytes needle_header = 1;
  270. bytes needle_body = 2;
  271. bool is_last_chunk = 3;
  272. }
  273. message VolumeTailReceiverRequest {
  274. uint32 volume_id = 1;
  275. uint64 since_ns = 2;
  276. uint32 idle_timeout_seconds = 3;
  277. string source_volume_server = 4;
  278. }
  279. message VolumeTailReceiverResponse {
  280. }
  281. message VolumeEcShardsGenerateRequest {
  282. uint32 volume_id = 1;
  283. string collection = 2;
  284. }
  285. message VolumeEcShardsGenerateResponse {
  286. }
  287. message VolumeEcShardsRebuildRequest {
  288. uint32 volume_id = 1;
  289. string collection = 2;
  290. }
  291. message VolumeEcShardsRebuildResponse {
  292. repeated uint32 rebuilt_shard_ids = 1;
  293. }
  294. message VolumeEcShardsCopyRequest {
  295. uint32 volume_id = 1;
  296. string collection = 2;
  297. repeated uint32 shard_ids = 3;
  298. bool copy_ecx_file = 4;
  299. string source_data_node = 5;
  300. bool copy_ecj_file = 6;
  301. bool copy_vif_file = 7;
  302. }
  303. message VolumeEcShardsCopyResponse {
  304. }
  305. message VolumeEcShardsDeleteRequest {
  306. uint32 volume_id = 1;
  307. string collection = 2;
  308. repeated uint32 shard_ids = 3;
  309. }
  310. message VolumeEcShardsDeleteResponse {
  311. }
  312. message VolumeEcShardsMountRequest {
  313. uint32 volume_id = 1;
  314. string collection = 2;
  315. repeated uint32 shard_ids = 3;
  316. }
  317. message VolumeEcShardsMountResponse {
  318. }
  319. message VolumeEcShardsUnmountRequest {
  320. uint32 volume_id = 1;
  321. repeated uint32 shard_ids = 3;
  322. }
  323. message VolumeEcShardsUnmountResponse {
  324. }
  325. message VolumeEcShardReadRequest {
  326. uint32 volume_id = 1;
  327. uint32 shard_id = 2;
  328. int64 offset = 3;
  329. int64 size = 4;
  330. uint64 file_key = 5;
  331. }
  332. message VolumeEcShardReadResponse {
  333. bytes data = 1;
  334. bool is_deleted = 2;
  335. }
  336. message VolumeEcBlobDeleteRequest {
  337. uint32 volume_id = 1;
  338. string collection = 2;
  339. uint64 file_key = 3;
  340. uint32 version = 4;
  341. }
  342. message VolumeEcBlobDeleteResponse {
  343. }
  344. message VolumeEcShardsToVolumeRequest {
  345. uint32 volume_id = 1;
  346. string collection = 2;
  347. }
  348. message VolumeEcShardsToVolumeResponse {
  349. }
  350. message ReadVolumeFileStatusRequest {
  351. uint32 volume_id = 1;
  352. }
  353. message ReadVolumeFileStatusResponse {
  354. uint32 volume_id = 1;
  355. uint64 idx_file_timestamp_seconds = 2;
  356. uint64 idx_file_size = 3;
  357. uint64 dat_file_timestamp_seconds = 4;
  358. uint64 dat_file_size = 5;
  359. uint64 file_count = 6;
  360. uint32 compaction_revision = 7;
  361. string collection = 8;
  362. string disk_type = 9;
  363. }
  364. message DiskStatus {
  365. string dir = 1;
  366. uint64 all = 2;
  367. uint64 used = 3;
  368. uint64 free = 4;
  369. float percent_free = 5;
  370. float percent_used = 6;
  371. string disk_type = 7;
  372. }
  373. message MemStatus {
  374. int32 goroutines = 1;
  375. uint64 all = 2;
  376. uint64 used = 3;
  377. uint64 free = 4;
  378. uint64 self = 5;
  379. uint64 heap = 6;
  380. uint64 stack = 7;
  381. }
  382. // tired storage on volume servers
  383. message RemoteFile {
  384. string backend_type = 1;
  385. string backend_id = 2;
  386. string key = 3;
  387. uint64 offset = 4;
  388. uint64 file_size = 5;
  389. uint64 modified_time = 6;
  390. string extension = 7;
  391. }
  392. message VolumeInfo {
  393. repeated RemoteFile files = 1;
  394. uint32 version = 2;
  395. string replication = 3;
  396. }
  397. // tiered storage
  398. message VolumeTierMoveDatToRemoteRequest {
  399. uint32 volume_id = 1;
  400. string collection = 2;
  401. string destination_backend_name = 3;
  402. bool keep_local_dat_file = 4;
  403. }
  404. message VolumeTierMoveDatToRemoteResponse {
  405. int64 processed = 1;
  406. float processedPercentage = 2;
  407. }
  408. message VolumeTierMoveDatFromRemoteRequest {
  409. uint32 volume_id = 1;
  410. string collection = 2;
  411. bool keep_remote_dat_file = 3;
  412. }
  413. message VolumeTierMoveDatFromRemoteResponse {
  414. int64 processed = 1;
  415. float processedPercentage = 2;
  416. }
  417. message VolumeServerStatusRequest {
  418. }
  419. message VolumeServerStatusResponse {
  420. repeated DiskStatus disk_statuses = 1;
  421. MemStatus memory_status = 2;
  422. string version = 3;
  423. string data_center = 4;
  424. string rack = 5;
  425. }
  426. message VolumeServerLeaveRequest {
  427. }
  428. message VolumeServerLeaveResponse {
  429. }
  430. // remote storage
  431. message FetchAndWriteNeedleRequest {
  432. uint32 volume_id = 1;
  433. uint64 needle_id = 2;
  434. uint32 cookie = 3;
  435. int64 offset = 4;
  436. int64 size = 5;
  437. message Replica {
  438. string url = 1;
  439. string public_url = 2;
  440. int32 grpc_port = 3;
  441. }
  442. repeated Replica replicas = 6;
  443. string auth = 7;
  444. // remote conf
  445. remote_pb.RemoteConf remote_conf = 15;
  446. remote_pb.RemoteStorageLocation remote_location = 16;
  447. }
  448. message FetchAndWriteNeedleResponse {
  449. }
  450. // select on volume servers
  451. message QueryRequest {
  452. repeated string selections = 1;
  453. repeated string from_file_ids = 2;
  454. message Filter {
  455. string field = 1;
  456. string operand = 2;
  457. string value = 3;
  458. }
  459. Filter filter = 3;
  460. message InputSerialization {
  461. // NONE | GZIP | BZIP2
  462. string compression_type = 1;
  463. message CSVInput {
  464. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  465. string record_delimiter = 2; // Default: \n
  466. string field_delimiter = 3; // Default: ,
  467. string quote_charactoer = 4; // Default: "
  468. string quote_escape_character = 5; // Default: "
  469. string comments = 6; // Default: #
  470. // If true, records might contain record delimiters within quote characters
  471. bool allow_quoted_record_delimiter = 7; // default False.
  472. }
  473. message JSONInput {
  474. string type = 1; // Valid values: DOCUMENT | LINES
  475. }
  476. message ParquetInput {
  477. }
  478. CSVInput csv_input = 2;
  479. JSONInput json_input = 3;
  480. ParquetInput parquet_input = 4;
  481. }
  482. InputSerialization input_serialization = 4;
  483. message OutputSerialization {
  484. message CSVOutput {
  485. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  486. string record_delimiter = 2; // Default: \n
  487. string field_delimiter = 3; // Default: ,
  488. string quote_charactoer = 4; // Default: "
  489. string quote_escape_character = 5; // Default: "
  490. }
  491. message JSONOutput {
  492. string record_delimiter = 1;
  493. }
  494. CSVOutput csv_output = 2;
  495. JSONOutput json_output = 3;
  496. }
  497. OutputSerialization output_serialization = 5;
  498. }
  499. message QueriedStripe {
  500. bytes records = 1;
  501. }
  502. message VolumeNeedleStatusRequest {
  503. uint32 volume_id = 1;
  504. uint64 needle_id = 2;
  505. }
  506. message VolumeNeedleStatusResponse {
  507. uint64 needle_id = 1;
  508. uint32 cookie = 2;
  509. uint32 size = 3;
  510. uint64 last_modified = 4;
  511. uint32 crc = 5;
  512. string ttl = 6;
  513. }
  514. message PingRequest {
  515. string target = 1; // default to ping itself
  516. string target_type = 2;
  517. }
  518. message PingResponse {
  519. int64 start_time_ns = 1;
  520. int64 remote_time_ns = 2;
  521. int64 stop_time_ns = 3;
  522. }