You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

506 lines
13 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb";
  4. //////////////////////////////////////////////////
  5. service VolumeServer {
  6. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  7. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  8. }
  9. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  10. }
  11. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
  12. }
  13. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  14. }
  15. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  16. }
  17. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  18. }
  19. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  20. }
  21. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  22. }
  23. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  24. }
  25. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  26. }
  27. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  28. }
  29. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  30. }
  31. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  32. }
  33. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  34. }
  35. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  36. }
  37. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  38. }
  39. // copy the .idx .dat files, and mount this volume
  40. rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
  41. }
  42. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  43. }
  44. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  45. }
  46. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  47. }
  48. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  49. }
  50. // erasure coding
  51. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  52. }
  53. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  54. }
  55. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  56. }
  57. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  58. }
  59. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  60. }
  61. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  62. }
  63. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  64. }
  65. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  66. }
  67. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  68. }
  69. // tiered storage
  70. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  71. }
  72. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  73. }
  74. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  75. }
  76. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  77. }
  78. // <experimental> query
  79. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  80. }
  81. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  82. }
  83. }
  84. //////////////////////////////////////////////////
  85. message BatchDeleteRequest {
  86. repeated string file_ids = 1;
  87. bool skip_cookie_check = 2;
  88. }
  89. message BatchDeleteResponse {
  90. repeated DeleteResult results = 1;
  91. }
  92. message DeleteResult {
  93. string file_id = 1;
  94. int32 status = 2;
  95. string error = 3;
  96. uint32 size = 4;
  97. uint32 version = 5;
  98. }
  99. message Empty {
  100. }
  101. message VacuumVolumeCheckRequest {
  102. uint32 volume_id = 1;
  103. }
  104. message VacuumVolumeCheckResponse {
  105. double garbage_ratio = 1;
  106. }
  107. message VacuumVolumeCompactRequest {
  108. uint32 volume_id = 1;
  109. int64 preallocate = 2;
  110. }
  111. message VacuumVolumeCompactResponse {
  112. }
  113. message VacuumVolumeCommitRequest {
  114. uint32 volume_id = 1;
  115. }
  116. message VacuumVolumeCommitResponse {
  117. bool is_read_only = 1;
  118. }
  119. message VacuumVolumeCleanupRequest {
  120. uint32 volume_id = 1;
  121. }
  122. message VacuumVolumeCleanupResponse {
  123. }
  124. message DeleteCollectionRequest {
  125. string collection = 1;
  126. }
  127. message DeleteCollectionResponse {
  128. }
  129. message AllocateVolumeRequest {
  130. uint32 volume_id = 1;
  131. string collection = 2;
  132. int64 preallocate = 3;
  133. string replication = 4;
  134. string ttl = 5;
  135. uint32 memory_map_max_size_mb = 6;
  136. string disk_type = 7;
  137. }
  138. message AllocateVolumeResponse {
  139. }
  140. message VolumeSyncStatusRequest {
  141. uint32 volume_id = 1;
  142. }
  143. message VolumeSyncStatusResponse {
  144. uint32 volume_id = 1;
  145. string collection = 2;
  146. string replication = 4;
  147. string ttl = 5;
  148. uint64 tail_offset = 6;
  149. uint32 compact_revision = 7;
  150. uint64 idx_file_size = 8;
  151. }
  152. message VolumeIncrementalCopyRequest {
  153. uint32 volume_id = 1;
  154. uint64 since_ns = 2;
  155. }
  156. message VolumeIncrementalCopyResponse {
  157. bytes file_content = 1;
  158. }
  159. message VolumeMountRequest {
  160. uint32 volume_id = 1;
  161. }
  162. message VolumeMountResponse {
  163. }
  164. message VolumeUnmountRequest {
  165. uint32 volume_id = 1;
  166. }
  167. message VolumeUnmountResponse {
  168. }
  169. message VolumeDeleteRequest {
  170. uint32 volume_id = 1;
  171. }
  172. message VolumeDeleteResponse {
  173. }
  174. message VolumeMarkReadonlyRequest {
  175. uint32 volume_id = 1;
  176. }
  177. message VolumeMarkReadonlyResponse {
  178. }
  179. message VolumeMarkWritableRequest {
  180. uint32 volume_id = 1;
  181. }
  182. message VolumeMarkWritableResponse {
  183. }
  184. message VolumeConfigureRequest {
  185. uint32 volume_id = 1;
  186. string replication = 2;
  187. }
  188. message VolumeConfigureResponse {
  189. string error = 1;
  190. }
  191. message VolumeStatusRequest {
  192. uint32 volume_id = 1;
  193. }
  194. message VolumeStatusResponse {
  195. bool is_read_only = 1;
  196. }
  197. message VolumeCopyRequest {
  198. uint32 volume_id = 1;
  199. string collection = 2;
  200. string replication = 3;
  201. string ttl = 4;
  202. string source_data_node = 5;
  203. }
  204. message VolumeCopyResponse {
  205. uint64 last_append_at_ns = 1;
  206. }
  207. message CopyFileRequest {
  208. uint32 volume_id = 1;
  209. string ext = 2;
  210. uint32 compaction_revision = 3;
  211. uint64 stop_offset = 4;
  212. string collection = 5;
  213. bool is_ec_volume = 6;
  214. bool ignore_source_file_not_found = 7;
  215. }
  216. message CopyFileResponse {
  217. bytes file_content = 1;
  218. }
  219. message VolumeTailSenderRequest {
  220. uint32 volume_id = 1;
  221. uint64 since_ns = 2;
  222. uint32 idle_timeout_seconds = 3;
  223. }
  224. message VolumeTailSenderResponse {
  225. bytes needle_header = 1;
  226. bytes needle_body = 2;
  227. bool is_last_chunk = 3;
  228. }
  229. message VolumeTailReceiverRequest {
  230. uint32 volume_id = 1;
  231. uint64 since_ns = 2;
  232. uint32 idle_timeout_seconds = 3;
  233. string source_volume_server = 4;
  234. }
  235. message VolumeTailReceiverResponse {
  236. }
  237. message VolumeEcShardsGenerateRequest {
  238. uint32 volume_id = 1;
  239. string collection = 2;
  240. }
  241. message VolumeEcShardsGenerateResponse {
  242. }
  243. message VolumeEcShardsRebuildRequest {
  244. uint32 volume_id = 1;
  245. string collection = 2;
  246. }
  247. message VolumeEcShardsRebuildResponse {
  248. repeated uint32 rebuilt_shard_ids = 1;
  249. }
  250. message VolumeEcShardsCopyRequest {
  251. uint32 volume_id = 1;
  252. string collection = 2;
  253. repeated uint32 shard_ids = 3;
  254. bool copy_ecx_file = 4;
  255. string source_data_node = 5;
  256. bool copy_ecj_file = 6;
  257. bool copy_vif_file = 7;
  258. }
  259. message VolumeEcShardsCopyResponse {
  260. }
  261. message VolumeEcShardsDeleteRequest {
  262. uint32 volume_id = 1;
  263. string collection = 2;
  264. repeated uint32 shard_ids = 3;
  265. }
  266. message VolumeEcShardsDeleteResponse {
  267. }
  268. message VolumeEcShardsMountRequest {
  269. uint32 volume_id = 1;
  270. string collection = 2;
  271. repeated uint32 shard_ids = 3;
  272. }
  273. message VolumeEcShardsMountResponse {
  274. }
  275. message VolumeEcShardsUnmountRequest {
  276. uint32 volume_id = 1;
  277. repeated uint32 shard_ids = 3;
  278. }
  279. message VolumeEcShardsUnmountResponse {
  280. }
  281. message VolumeEcShardReadRequest {
  282. uint32 volume_id = 1;
  283. uint32 shard_id = 2;
  284. int64 offset = 3;
  285. int64 size = 4;
  286. uint64 file_key = 5;
  287. }
  288. message VolumeEcShardReadResponse {
  289. bytes data = 1;
  290. bool is_deleted = 2;
  291. }
  292. message VolumeEcBlobDeleteRequest {
  293. uint32 volume_id = 1;
  294. string collection = 2;
  295. uint64 file_key = 3;
  296. uint32 version = 4;
  297. }
  298. message VolumeEcBlobDeleteResponse {
  299. }
  300. message VolumeEcShardsToVolumeRequest {
  301. uint32 volume_id = 1;
  302. string collection = 2;
  303. }
  304. message VolumeEcShardsToVolumeResponse {
  305. }
  306. message ReadVolumeFileStatusRequest {
  307. uint32 volume_id = 1;
  308. }
  309. message ReadVolumeFileStatusResponse {
  310. uint32 volume_id = 1;
  311. uint64 idx_file_timestamp_seconds = 2;
  312. uint64 idx_file_size = 3;
  313. uint64 dat_file_timestamp_seconds = 4;
  314. uint64 dat_file_size = 5;
  315. uint64 file_count = 6;
  316. uint32 compaction_revision = 7;
  317. string collection = 8;
  318. string disk_type = 9;
  319. }
  320. message DiskStatus {
  321. string dir = 1;
  322. uint64 all = 2;
  323. uint64 used = 3;
  324. uint64 free = 4;
  325. float percent_free = 5;
  326. float percent_used = 6;
  327. }
  328. message MemStatus {
  329. int32 goroutines = 1;
  330. uint64 all = 2;
  331. uint64 used = 3;
  332. uint64 free = 4;
  333. uint64 self = 5;
  334. uint64 heap = 6;
  335. uint64 stack = 7;
  336. }
  337. // tired storage on volume servers
  338. message RemoteFile {
  339. string backend_type = 1;
  340. string backend_id = 2;
  341. string key = 3;
  342. uint64 offset = 4;
  343. uint64 file_size = 5;
  344. uint64 modified_time = 6;
  345. string extension = 7;
  346. }
  347. message VolumeInfo {
  348. repeated RemoteFile files = 1;
  349. uint32 version = 2;
  350. string replication = 3;
  351. }
  352. message VolumeTierMoveDatToRemoteRequest {
  353. uint32 volume_id = 1;
  354. string collection = 2;
  355. string destination_backend_name = 3;
  356. bool keep_local_dat_file = 4;
  357. }
  358. message VolumeTierMoveDatToRemoteResponse {
  359. int64 processed = 1;
  360. float processedPercentage = 2;
  361. }
  362. message VolumeTierMoveDatFromRemoteRequest {
  363. uint32 volume_id = 1;
  364. string collection = 2;
  365. bool keep_remote_dat_file = 3;
  366. }
  367. message VolumeTierMoveDatFromRemoteResponse {
  368. int64 processed = 1;
  369. float processedPercentage = 2;
  370. }
  371. message VolumeServerStatusRequest {
  372. }
  373. message VolumeServerStatusResponse {
  374. repeated DiskStatus disk_statuses = 1;
  375. MemStatus memory_status = 2;
  376. }
  377. message VolumeServerLeaveRequest {
  378. }
  379. message VolumeServerLeaveResponse {
  380. }
  381. // select on volume servers
  382. message QueryRequest {
  383. repeated string selections = 1;
  384. repeated string from_file_ids = 2;
  385. message Filter {
  386. string field = 1;
  387. string operand = 2;
  388. string value = 3;
  389. }
  390. Filter filter = 3;
  391. message InputSerialization {
  392. // NONE | GZIP | BZIP2
  393. string compression_type = 1;
  394. message CSVInput {
  395. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  396. string record_delimiter = 2; // Default: \n
  397. string field_delimiter = 3; // Default: ,
  398. string quote_charactoer = 4; // Default: "
  399. string quote_escape_character = 5; // Default: "
  400. string comments = 6; // Default: #
  401. // If true, records might contain record delimiters within quote characters
  402. bool allow_quoted_record_delimiter = 7; // default False.
  403. }
  404. message JSONInput {
  405. string type = 1; // Valid values: DOCUMENT | LINES
  406. }
  407. message ParquetInput {
  408. }
  409. CSVInput csv_input = 2;
  410. JSONInput json_input = 3;
  411. ParquetInput parquet_input = 4;
  412. }
  413. InputSerialization input_serialization = 4;
  414. message OutputSerialization {
  415. message CSVOutput {
  416. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  417. string record_delimiter = 2; // Default: \n
  418. string field_delimiter = 3; // Default: ,
  419. string quote_charactoer = 4; // Default: "
  420. string quote_escape_character = 5; // Default: "
  421. }
  422. message JSONOutput {
  423. string record_delimiter = 1;
  424. }
  425. CSVOutput csv_output = 2;
  426. JSONOutput json_output = 3;
  427. }
  428. OutputSerialization output_serialization = 5;
  429. }
  430. message QueriedStripe {
  431. bytes records = 1;
  432. }
  433. message VolumeNeedleStatusRequest {
  434. uint32 volume_id = 1;
  435. uint64 needle_id = 2;
  436. }
  437. message VolumeNeedleStatusResponse {
  438. uint64 needle_id = 1;
  439. uint32 cookie = 2;
  440. uint32 size = 3;
  441. uint64 last_modified = 4;
  442. uint32 crc = 5;
  443. string ttl = 6;
  444. }