You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

592 lines
15 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
3 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb";
  4. import "remote.proto";
  5. //////////////////////////////////////////////////
  6. service VolumeServer {
  7. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  8. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  9. }
  10. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  11. }
  12. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
  13. }
  14. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  15. }
  16. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  17. }
  18. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  19. }
  20. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  21. }
  22. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  23. }
  24. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  25. }
  26. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  27. }
  28. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  29. }
  30. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  31. }
  32. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  33. }
  34. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  35. }
  36. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  37. }
  38. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  39. }
  40. // copy the .idx .dat files, and mount this volume
  41. rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
  42. }
  43. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  44. }
  45. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  46. }
  47. rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
  48. }
  49. rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
  50. }
  51. rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
  52. }
  53. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  54. }
  55. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  56. }
  57. // erasure coding
  58. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  59. }
  60. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  61. }
  62. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  63. }
  64. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  65. }
  66. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  67. }
  68. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  69. }
  70. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  71. }
  72. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  73. }
  74. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  75. }
  76. // tiered storage
  77. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  78. }
  79. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  80. }
  81. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  82. }
  83. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  84. }
  85. // remote storage
  86. rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
  87. }
  88. // <experimental> query
  89. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  90. }
  91. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  92. }
  93. rpc Ping (PingRequest) returns (PingResponse) {
  94. }
  95. }
  96. //////////////////////////////////////////////////
  97. message BatchDeleteRequest {
  98. repeated string file_ids = 1;
  99. bool skip_cookie_check = 2;
  100. }
  101. message BatchDeleteResponse {
  102. repeated DeleteResult results = 1;
  103. }
  104. message DeleteResult {
  105. string file_id = 1;
  106. int32 status = 2;
  107. string error = 3;
  108. uint32 size = 4;
  109. uint32 version = 5;
  110. }
  111. message Empty {
  112. }
  113. message VacuumVolumeCheckRequest {
  114. uint32 volume_id = 1;
  115. }
  116. message VacuumVolumeCheckResponse {
  117. double garbage_ratio = 1;
  118. }
  119. message VacuumVolumeCompactRequest {
  120. uint32 volume_id = 1;
  121. int64 preallocate = 2;
  122. }
  123. message VacuumVolumeCompactResponse {
  124. int64 processed_bytes = 1;
  125. }
  126. message VacuumVolumeCommitRequest {
  127. uint32 volume_id = 1;
  128. }
  129. message VacuumVolumeCommitResponse {
  130. bool is_read_only = 1;
  131. }
  132. message VacuumVolumeCleanupRequest {
  133. uint32 volume_id = 1;
  134. }
  135. message VacuumVolumeCleanupResponse {
  136. }
  137. message DeleteCollectionRequest {
  138. string collection = 1;
  139. }
  140. message DeleteCollectionResponse {
  141. }
  142. message AllocateVolumeRequest {
  143. uint32 volume_id = 1;
  144. string collection = 2;
  145. int64 preallocate = 3;
  146. string replication = 4;
  147. string ttl = 5;
  148. uint32 memory_map_max_size_mb = 6;
  149. string disk_type = 7;
  150. }
  151. message AllocateVolumeResponse {
  152. }
  153. message VolumeSyncStatusRequest {
  154. uint32 volume_id = 1;
  155. }
  156. message VolumeSyncStatusResponse {
  157. uint32 volume_id = 1;
  158. string collection = 2;
  159. string replication = 4;
  160. string ttl = 5;
  161. uint64 tail_offset = 6;
  162. uint32 compact_revision = 7;
  163. uint64 idx_file_size = 8;
  164. }
  165. message VolumeIncrementalCopyRequest {
  166. uint32 volume_id = 1;
  167. uint64 since_ns = 2;
  168. }
  169. message VolumeIncrementalCopyResponse {
  170. bytes file_content = 1;
  171. }
  172. message VolumeMountRequest {
  173. uint32 volume_id = 1;
  174. }
  175. message VolumeMountResponse {
  176. }
  177. message VolumeUnmountRequest {
  178. uint32 volume_id = 1;
  179. }
  180. message VolumeUnmountResponse {
  181. }
  182. message VolumeDeleteRequest {
  183. uint32 volume_id = 1;
  184. }
  185. message VolumeDeleteResponse {
  186. }
  187. message VolumeMarkReadonlyRequest {
  188. uint32 volume_id = 1;
  189. }
  190. message VolumeMarkReadonlyResponse {
  191. }
  192. message VolumeMarkWritableRequest {
  193. uint32 volume_id = 1;
  194. }
  195. message VolumeMarkWritableResponse {
  196. }
  197. message VolumeConfigureRequest {
  198. uint32 volume_id = 1;
  199. string replication = 2;
  200. }
  201. message VolumeConfigureResponse {
  202. string error = 1;
  203. }
  204. message VolumeStatusRequest {
  205. uint32 volume_id = 1;
  206. }
  207. message VolumeStatusResponse {
  208. bool is_read_only = 1;
  209. }
  210. message VolumeCopyRequest {
  211. uint32 volume_id = 1;
  212. string collection = 2;
  213. string replication = 3;
  214. string ttl = 4;
  215. string source_data_node = 5;
  216. string disk_type = 6;
  217. }
  218. message VolumeCopyResponse {
  219. uint64 last_append_at_ns = 1;
  220. int64 processed_bytes = 2;
  221. }
  222. message CopyFileRequest {
  223. uint32 volume_id = 1;
  224. string ext = 2;
  225. uint32 compaction_revision = 3;
  226. uint64 stop_offset = 4;
  227. string collection = 5;
  228. bool is_ec_volume = 6;
  229. bool ignore_source_file_not_found = 7;
  230. }
  231. message CopyFileResponse {
  232. bytes file_content = 1;
  233. int64 modified_ts_ns = 2;
  234. }
  235. message ReadNeedleBlobRequest {
  236. uint32 volume_id = 1;
  237. uint64 needle_id = 2;
  238. int64 offset = 3; // actual offset
  239. int32 size = 4;
  240. }
  241. message ReadNeedleBlobResponse {
  242. bytes needle_blob = 1;
  243. }
  244. message WriteNeedleBlobRequest {
  245. uint32 volume_id = 1;
  246. uint64 needle_id = 2;
  247. int32 size = 3;
  248. bytes needle_blob = 4;
  249. }
  250. message WriteNeedleBlobResponse {
  251. }
  252. message ReadAllNeedlesRequest {
  253. repeated uint32 volume_ids = 1;
  254. }
  255. message ReadAllNeedlesResponse {
  256. uint32 volume_id = 1;
  257. uint64 needle_id = 2;
  258. uint32 cookie = 3;
  259. bytes needle_blob = 5;
  260. }
  261. message VolumeTailSenderRequest {
  262. uint32 volume_id = 1;
  263. uint64 since_ns = 2;
  264. uint32 idle_timeout_seconds = 3;
  265. }
  266. message VolumeTailSenderResponse {
  267. bytes needle_header = 1;
  268. bytes needle_body = 2;
  269. bool is_last_chunk = 3;
  270. }
  271. message VolumeTailReceiverRequest {
  272. uint32 volume_id = 1;
  273. uint64 since_ns = 2;
  274. uint32 idle_timeout_seconds = 3;
  275. string source_volume_server = 4;
  276. }
  277. message VolumeTailReceiverResponse {
  278. }
  279. message VolumeEcShardsGenerateRequest {
  280. uint32 volume_id = 1;
  281. string collection = 2;
  282. }
  283. message VolumeEcShardsGenerateResponse {
  284. }
  285. message VolumeEcShardsRebuildRequest {
  286. uint32 volume_id = 1;
  287. string collection = 2;
  288. }
  289. message VolumeEcShardsRebuildResponse {
  290. repeated uint32 rebuilt_shard_ids = 1;
  291. }
  292. message VolumeEcShardsCopyRequest {
  293. uint32 volume_id = 1;
  294. string collection = 2;
  295. repeated uint32 shard_ids = 3;
  296. bool copy_ecx_file = 4;
  297. string source_data_node = 5;
  298. bool copy_ecj_file = 6;
  299. bool copy_vif_file = 7;
  300. }
  301. message VolumeEcShardsCopyResponse {
  302. }
  303. message VolumeEcShardsDeleteRequest {
  304. uint32 volume_id = 1;
  305. string collection = 2;
  306. repeated uint32 shard_ids = 3;
  307. }
  308. message VolumeEcShardsDeleteResponse {
  309. }
  310. message VolumeEcShardsMountRequest {
  311. uint32 volume_id = 1;
  312. string collection = 2;
  313. repeated uint32 shard_ids = 3;
  314. }
  315. message VolumeEcShardsMountResponse {
  316. }
  317. message VolumeEcShardsUnmountRequest {
  318. uint32 volume_id = 1;
  319. repeated uint32 shard_ids = 3;
  320. }
  321. message VolumeEcShardsUnmountResponse {
  322. }
  323. message VolumeEcShardReadRequest {
  324. uint32 volume_id = 1;
  325. uint32 shard_id = 2;
  326. int64 offset = 3;
  327. int64 size = 4;
  328. uint64 file_key = 5;
  329. }
  330. message VolumeEcShardReadResponse {
  331. bytes data = 1;
  332. bool is_deleted = 2;
  333. }
  334. message VolumeEcBlobDeleteRequest {
  335. uint32 volume_id = 1;
  336. string collection = 2;
  337. uint64 file_key = 3;
  338. uint32 version = 4;
  339. }
  340. message VolumeEcBlobDeleteResponse {
  341. }
  342. message VolumeEcShardsToVolumeRequest {
  343. uint32 volume_id = 1;
  344. string collection = 2;
  345. }
  346. message VolumeEcShardsToVolumeResponse {
  347. }
  348. message ReadVolumeFileStatusRequest {
  349. uint32 volume_id = 1;
  350. }
  351. message ReadVolumeFileStatusResponse {
  352. uint32 volume_id = 1;
  353. uint64 idx_file_timestamp_seconds = 2;
  354. uint64 idx_file_size = 3;
  355. uint64 dat_file_timestamp_seconds = 4;
  356. uint64 dat_file_size = 5;
  357. uint64 file_count = 6;
  358. uint32 compaction_revision = 7;
  359. string collection = 8;
  360. string disk_type = 9;
  361. }
  362. message DiskStatus {
  363. string dir = 1;
  364. uint64 all = 2;
  365. uint64 used = 3;
  366. uint64 free = 4;
  367. float percent_free = 5;
  368. float percent_used = 6;
  369. string disk_type = 7;
  370. }
  371. message MemStatus {
  372. int32 goroutines = 1;
  373. uint64 all = 2;
  374. uint64 used = 3;
  375. uint64 free = 4;
  376. uint64 self = 5;
  377. uint64 heap = 6;
  378. uint64 stack = 7;
  379. }
  380. // tired storage on volume servers
  381. message RemoteFile {
  382. string backend_type = 1;
  383. string backend_id = 2;
  384. string key = 3;
  385. uint64 offset = 4;
  386. uint64 file_size = 5;
  387. uint64 modified_time = 6;
  388. string extension = 7;
  389. }
  390. message VolumeInfo {
  391. repeated RemoteFile files = 1;
  392. uint32 version = 2;
  393. string replication = 3;
  394. }
  395. // tiered storage
  396. message VolumeTierMoveDatToRemoteRequest {
  397. uint32 volume_id = 1;
  398. string collection = 2;
  399. string destination_backend_name = 3;
  400. bool keep_local_dat_file = 4;
  401. }
  402. message VolumeTierMoveDatToRemoteResponse {
  403. int64 processed = 1;
  404. float processedPercentage = 2;
  405. }
  406. message VolumeTierMoveDatFromRemoteRequest {
  407. uint32 volume_id = 1;
  408. string collection = 2;
  409. bool keep_remote_dat_file = 3;
  410. }
  411. message VolumeTierMoveDatFromRemoteResponse {
  412. int64 processed = 1;
  413. float processedPercentage = 2;
  414. }
  415. message VolumeServerStatusRequest {
  416. }
  417. message VolumeServerStatusResponse {
  418. repeated DiskStatus disk_statuses = 1;
  419. MemStatus memory_status = 2;
  420. string version = 3;
  421. string data_center = 4;
  422. string rack = 5;
  423. }
  424. message VolumeServerLeaveRequest {
  425. }
  426. message VolumeServerLeaveResponse {
  427. }
  428. // remote storage
  429. message FetchAndWriteNeedleRequest {
  430. uint32 volume_id = 1;
  431. uint64 needle_id = 2;
  432. uint32 cookie = 3;
  433. int64 offset = 4;
  434. int64 size = 5;
  435. message Replica {
  436. string url = 1;
  437. string public_url = 2;
  438. int32 grpc_port = 3;
  439. }
  440. repeated Replica replicas = 6;
  441. string auth = 7;
  442. // remote conf
  443. remote_pb.RemoteConf remote_conf = 15;
  444. remote_pb.RemoteStorageLocation remote_location = 16;
  445. }
  446. message FetchAndWriteNeedleResponse {
  447. }
  448. // select on volume servers
  449. message QueryRequest {
  450. repeated string selections = 1;
  451. repeated string from_file_ids = 2;
  452. message Filter {
  453. string field = 1;
  454. string operand = 2;
  455. string value = 3;
  456. }
  457. Filter filter = 3;
  458. message InputSerialization {
  459. // NONE | GZIP | BZIP2
  460. string compression_type = 1;
  461. message CSVInput {
  462. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  463. string record_delimiter = 2; // Default: \n
  464. string field_delimiter = 3; // Default: ,
  465. string quote_charactoer = 4; // Default: "
  466. string quote_escape_character = 5; // Default: "
  467. string comments = 6; // Default: #
  468. // If true, records might contain record delimiters within quote characters
  469. bool allow_quoted_record_delimiter = 7; // default False.
  470. }
  471. message JSONInput {
  472. string type = 1; // Valid values: DOCUMENT | LINES
  473. }
  474. message ParquetInput {
  475. }
  476. CSVInput csv_input = 2;
  477. JSONInput json_input = 3;
  478. ParquetInput parquet_input = 4;
  479. }
  480. InputSerialization input_serialization = 4;
  481. message OutputSerialization {
  482. message CSVOutput {
  483. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  484. string record_delimiter = 2; // Default: \n
  485. string field_delimiter = 3; // Default: ,
  486. string quote_charactoer = 4; // Default: "
  487. string quote_escape_character = 5; // Default: "
  488. }
  489. message JSONOutput {
  490. string record_delimiter = 1;
  491. }
  492. CSVOutput csv_output = 2;
  493. JSONOutput json_output = 3;
  494. }
  495. OutputSerialization output_serialization = 5;
  496. }
  497. message QueriedStripe {
  498. bytes records = 1;
  499. }
  500. message VolumeNeedleStatusRequest {
  501. uint32 volume_id = 1;
  502. uint64 needle_id = 2;
  503. }
  504. message VolumeNeedleStatusResponse {
  505. uint64 needle_id = 1;
  506. uint32 cookie = 2;
  507. uint32 size = 3;
  508. uint64 last_modified = 4;
  509. uint32 crc = 5;
  510. string ttl = 6;
  511. }
  512. message PingRequest {
  513. string target = 1; // default to ping itself
  514. string target_type = 2;
  515. }
  516. message PingResponse {
  517. int64 start_time_ns = 1;
  518. int64 remote_time_ns = 2;
  519. int64 stop_time_ns = 3;
  520. }