You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

557 lines
14 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
3 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
3 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb";
  4. //////////////////////////////////////////////////
  5. service VolumeServer {
  6. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  7. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  8. }
  9. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  10. }
  11. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
  12. }
  13. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  14. }
  15. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  16. }
  17. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  18. }
  19. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  20. }
  21. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  22. }
  23. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  24. }
  25. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  26. }
  27. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  28. }
  29. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  30. }
  31. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  32. }
  33. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  34. }
  35. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  36. }
  37. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  38. }
  39. // copy the .idx .dat files, and mount this volume
  40. rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
  41. }
  42. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  43. }
  44. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  45. }
  46. rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
  47. }
  48. rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
  49. }
  50. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  51. }
  52. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  53. }
  54. // erasure coding
  55. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  56. }
  57. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  58. }
  59. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  60. }
  61. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  62. }
  63. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  64. }
  65. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  66. }
  67. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  68. }
  69. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  70. }
  71. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  72. }
  73. // tiered storage
  74. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  75. }
  76. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  77. }
  78. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  79. }
  80. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  81. }
  82. // remote storage
  83. rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
  84. }
  85. // <experimental> query
  86. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  87. }
  88. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  89. }
  90. }
  91. //////////////////////////////////////////////////
  92. message BatchDeleteRequest {
  93. repeated string file_ids = 1;
  94. bool skip_cookie_check = 2;
  95. }
  96. message BatchDeleteResponse {
  97. repeated DeleteResult results = 1;
  98. }
  99. message DeleteResult {
  100. string file_id = 1;
  101. int32 status = 2;
  102. string error = 3;
  103. uint32 size = 4;
  104. uint32 version = 5;
  105. }
  106. message Empty {
  107. }
  108. message VacuumVolumeCheckRequest {
  109. uint32 volume_id = 1;
  110. }
  111. message VacuumVolumeCheckResponse {
  112. double garbage_ratio = 1;
  113. }
  114. message VacuumVolumeCompactRequest {
  115. uint32 volume_id = 1;
  116. int64 preallocate = 2;
  117. }
  118. message VacuumVolumeCompactResponse {
  119. }
  120. message VacuumVolumeCommitRequest {
  121. uint32 volume_id = 1;
  122. }
  123. message VacuumVolumeCommitResponse {
  124. bool is_read_only = 1;
  125. }
  126. message VacuumVolumeCleanupRequest {
  127. uint32 volume_id = 1;
  128. }
  129. message VacuumVolumeCleanupResponse {
  130. }
  131. message DeleteCollectionRequest {
  132. string collection = 1;
  133. }
  134. message DeleteCollectionResponse {
  135. }
  136. message AllocateVolumeRequest {
  137. uint32 volume_id = 1;
  138. string collection = 2;
  139. int64 preallocate = 3;
  140. string replication = 4;
  141. string ttl = 5;
  142. uint32 memory_map_max_size_mb = 6;
  143. string disk_type = 7;
  144. }
  145. message AllocateVolumeResponse {
  146. }
  147. message VolumeSyncStatusRequest {
  148. uint32 volume_id = 1;
  149. }
  150. message VolumeSyncStatusResponse {
  151. uint32 volume_id = 1;
  152. string collection = 2;
  153. string replication = 4;
  154. string ttl = 5;
  155. uint64 tail_offset = 6;
  156. uint32 compact_revision = 7;
  157. uint64 idx_file_size = 8;
  158. }
  159. message VolumeIncrementalCopyRequest {
  160. uint32 volume_id = 1;
  161. uint64 since_ns = 2;
  162. }
  163. message VolumeIncrementalCopyResponse {
  164. bytes file_content = 1;
  165. }
  166. message VolumeMountRequest {
  167. uint32 volume_id = 1;
  168. }
  169. message VolumeMountResponse {
  170. }
  171. message VolumeUnmountRequest {
  172. uint32 volume_id = 1;
  173. }
  174. message VolumeUnmountResponse {
  175. }
  176. message VolumeDeleteRequest {
  177. uint32 volume_id = 1;
  178. }
  179. message VolumeDeleteResponse {
  180. }
  181. message VolumeMarkReadonlyRequest {
  182. uint32 volume_id = 1;
  183. }
  184. message VolumeMarkReadonlyResponse {
  185. }
  186. message VolumeMarkWritableRequest {
  187. uint32 volume_id = 1;
  188. }
  189. message VolumeMarkWritableResponse {
  190. }
  191. message VolumeConfigureRequest {
  192. uint32 volume_id = 1;
  193. string replication = 2;
  194. }
  195. message VolumeConfigureResponse {
  196. string error = 1;
  197. }
  198. message VolumeStatusRequest {
  199. uint32 volume_id = 1;
  200. }
  201. message VolumeStatusResponse {
  202. bool is_read_only = 1;
  203. }
  204. message VolumeCopyRequest {
  205. uint32 volume_id = 1;
  206. string collection = 2;
  207. string replication = 3;
  208. string ttl = 4;
  209. string source_data_node = 5;
  210. string disk_type = 6;
  211. }
  212. message VolumeCopyResponse {
  213. uint64 last_append_at_ns = 1;
  214. }
  215. message CopyFileRequest {
  216. uint32 volume_id = 1;
  217. string ext = 2;
  218. uint32 compaction_revision = 3;
  219. uint64 stop_offset = 4;
  220. string collection = 5;
  221. bool is_ec_volume = 6;
  222. bool ignore_source_file_not_found = 7;
  223. }
  224. message CopyFileResponse {
  225. bytes file_content = 1;
  226. }
  227. message ReadNeedleBlobRequest {
  228. uint32 volume_id = 1;
  229. uint64 needle_id = 2;
  230. int64 offset = 3; // actual offset
  231. int32 size = 4;
  232. }
  233. message ReadNeedleBlobResponse {
  234. bytes needle_blob = 1;
  235. }
  236. message WriteNeedleBlobRequest {
  237. uint32 volume_id = 1;
  238. uint64 needle_id = 2;
  239. int32 size = 3;
  240. bytes needle_blob = 4;
  241. }
  242. message WriteNeedleBlobResponse {
  243. }
  244. message VolumeTailSenderRequest {
  245. uint32 volume_id = 1;
  246. uint64 since_ns = 2;
  247. uint32 idle_timeout_seconds = 3;
  248. }
  249. message VolumeTailSenderResponse {
  250. bytes needle_header = 1;
  251. bytes needle_body = 2;
  252. bool is_last_chunk = 3;
  253. }
  254. message VolumeTailReceiverRequest {
  255. uint32 volume_id = 1;
  256. uint64 since_ns = 2;
  257. uint32 idle_timeout_seconds = 3;
  258. string source_volume_server = 4;
  259. }
  260. message VolumeTailReceiverResponse {
  261. }
  262. message VolumeEcShardsGenerateRequest {
  263. uint32 volume_id = 1;
  264. string collection = 2;
  265. }
  266. message VolumeEcShardsGenerateResponse {
  267. }
  268. message VolumeEcShardsRebuildRequest {
  269. uint32 volume_id = 1;
  270. string collection = 2;
  271. }
  272. message VolumeEcShardsRebuildResponse {
  273. repeated uint32 rebuilt_shard_ids = 1;
  274. }
  275. message VolumeEcShardsCopyRequest {
  276. uint32 volume_id = 1;
  277. string collection = 2;
  278. repeated uint32 shard_ids = 3;
  279. bool copy_ecx_file = 4;
  280. string source_data_node = 5;
  281. bool copy_ecj_file = 6;
  282. bool copy_vif_file = 7;
  283. }
  284. message VolumeEcShardsCopyResponse {
  285. }
  286. message VolumeEcShardsDeleteRequest {
  287. uint32 volume_id = 1;
  288. string collection = 2;
  289. repeated uint32 shard_ids = 3;
  290. }
  291. message VolumeEcShardsDeleteResponse {
  292. }
  293. message VolumeEcShardsMountRequest {
  294. uint32 volume_id = 1;
  295. string collection = 2;
  296. repeated uint32 shard_ids = 3;
  297. }
  298. message VolumeEcShardsMountResponse {
  299. }
  300. message VolumeEcShardsUnmountRequest {
  301. uint32 volume_id = 1;
  302. repeated uint32 shard_ids = 3;
  303. }
  304. message VolumeEcShardsUnmountResponse {
  305. }
  306. message VolumeEcShardReadRequest {
  307. uint32 volume_id = 1;
  308. uint32 shard_id = 2;
  309. int64 offset = 3;
  310. int64 size = 4;
  311. uint64 file_key = 5;
  312. }
  313. message VolumeEcShardReadResponse {
  314. bytes data = 1;
  315. bool is_deleted = 2;
  316. }
  317. message VolumeEcBlobDeleteRequest {
  318. uint32 volume_id = 1;
  319. string collection = 2;
  320. uint64 file_key = 3;
  321. uint32 version = 4;
  322. }
  323. message VolumeEcBlobDeleteResponse {
  324. }
  325. message VolumeEcShardsToVolumeRequest {
  326. uint32 volume_id = 1;
  327. string collection = 2;
  328. }
  329. message VolumeEcShardsToVolumeResponse {
  330. }
  331. message ReadVolumeFileStatusRequest {
  332. uint32 volume_id = 1;
  333. }
  334. message ReadVolumeFileStatusResponse {
  335. uint32 volume_id = 1;
  336. uint64 idx_file_timestamp_seconds = 2;
  337. uint64 idx_file_size = 3;
  338. uint64 dat_file_timestamp_seconds = 4;
  339. uint64 dat_file_size = 5;
  340. uint64 file_count = 6;
  341. uint32 compaction_revision = 7;
  342. string collection = 8;
  343. string disk_type = 9;
  344. }
  345. message DiskStatus {
  346. string dir = 1;
  347. uint64 all = 2;
  348. uint64 used = 3;
  349. uint64 free = 4;
  350. float percent_free = 5;
  351. float percent_used = 6;
  352. string disk_type = 7;
  353. }
  354. message MemStatus {
  355. int32 goroutines = 1;
  356. uint64 all = 2;
  357. uint64 used = 3;
  358. uint64 free = 4;
  359. uint64 self = 5;
  360. uint64 heap = 6;
  361. uint64 stack = 7;
  362. }
  363. // tired storage on volume servers
  364. message RemoteFile {
  365. string backend_type = 1;
  366. string backend_id = 2;
  367. string key = 3;
  368. uint64 offset = 4;
  369. uint64 file_size = 5;
  370. uint64 modified_time = 6;
  371. string extension = 7;
  372. }
  373. message VolumeInfo {
  374. repeated RemoteFile files = 1;
  375. uint32 version = 2;
  376. string replication = 3;
  377. }
  378. // tiered storage
  379. message VolumeTierMoveDatToRemoteRequest {
  380. uint32 volume_id = 1;
  381. string collection = 2;
  382. string destination_backend_name = 3;
  383. bool keep_local_dat_file = 4;
  384. }
  385. message VolumeTierMoveDatToRemoteResponse {
  386. int64 processed = 1;
  387. float processedPercentage = 2;
  388. }
  389. message VolumeTierMoveDatFromRemoteRequest {
  390. uint32 volume_id = 1;
  391. string collection = 2;
  392. bool keep_remote_dat_file = 3;
  393. }
  394. message VolumeTierMoveDatFromRemoteResponse {
  395. int64 processed = 1;
  396. float processedPercentage = 2;
  397. }
  398. message VolumeServerStatusRequest {
  399. }
  400. message VolumeServerStatusResponse {
  401. repeated DiskStatus disk_statuses = 1;
  402. MemStatus memory_status = 2;
  403. }
  404. message VolumeServerLeaveRequest {
  405. }
  406. message VolumeServerLeaveResponse {
  407. }
  408. // remote storage
  409. message FetchAndWriteNeedleRequest {
  410. uint32 volume_id = 1;
  411. uint64 needle_id = 2;
  412. uint32 cookie = 3;
  413. int64 offset = 4;
  414. int64 size = 5;
  415. // remote conf
  416. string remote_type = 6;
  417. string remote_name = 7;
  418. string s3_access_key = 8;
  419. string s3_secret_key = 9;
  420. string s3_region = 10;
  421. string s3_endpoint = 11;
  422. string remote_bucket = 12;
  423. string remote_path = 13;
  424. }
  425. message FetchAndWriteNeedleResponse {
  426. }
  427. // select on volume servers
  428. message QueryRequest {
  429. repeated string selections = 1;
  430. repeated string from_file_ids = 2;
  431. message Filter {
  432. string field = 1;
  433. string operand = 2;
  434. string value = 3;
  435. }
  436. Filter filter = 3;
  437. message InputSerialization {
  438. // NONE | GZIP | BZIP2
  439. string compression_type = 1;
  440. message CSVInput {
  441. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  442. string record_delimiter = 2; // Default: \n
  443. string field_delimiter = 3; // Default: ,
  444. string quote_charactoer = 4; // Default: "
  445. string quote_escape_character = 5; // Default: "
  446. string comments = 6; // Default: #
  447. // If true, records might contain record delimiters within quote characters
  448. bool allow_quoted_record_delimiter = 7; // default False.
  449. }
  450. message JSONInput {
  451. string type = 1; // Valid values: DOCUMENT | LINES
  452. }
  453. message ParquetInput {
  454. }
  455. CSVInput csv_input = 2;
  456. JSONInput json_input = 3;
  457. ParquetInput parquet_input = 4;
  458. }
  459. InputSerialization input_serialization = 4;
  460. message OutputSerialization {
  461. message CSVOutput {
  462. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  463. string record_delimiter = 2; // Default: \n
  464. string field_delimiter = 3; // Default: ,
  465. string quote_charactoer = 4; // Default: "
  466. string quote_escape_character = 5; // Default: "
  467. }
  468. message JSONOutput {
  469. string record_delimiter = 1;
  470. }
  471. CSVOutput csv_output = 2;
  472. JSONOutput json_output = 3;
  473. }
  474. OutputSerialization output_serialization = 5;
  475. }
  476. message QueriedStripe {
  477. bytes records = 1;
  478. }
  479. message VolumeNeedleStatusRequest {
  480. uint32 volume_id = 1;
  481. uint64 needle_id = 2;
  482. }
  483. message VolumeNeedleStatusResponse {
  484. uint64 needle_id = 1;
  485. uint32 cookie = 2;
  486. uint32 size = 3;
  487. uint64 last_modified = 4;
  488. uint32 crc = 5;
  489. string ttl = 6;
  490. }