You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

507 lines
13 KiB

5 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
  1. syntax = "proto3";
  2. package volume_server_pb;
  3. option go_package = "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb";
  4. //////////////////////////////////////////////////
  5. service VolumeServer {
  6. //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
  7. rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
  8. }
  9. rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
  10. }
  11. rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
  12. }
  13. rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
  14. }
  15. rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
  16. }
  17. rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
  18. }
  19. rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
  20. }
  21. rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
  22. }
  23. rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
  24. }
  25. rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
  26. }
  27. rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
  28. }
  29. rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
  30. }
  31. rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
  32. }
  33. rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
  34. }
  35. rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
  36. }
  37. rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
  38. }
  39. // copy the .idx .dat files, and mount this volume
  40. rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
  41. }
  42. rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
  43. }
  44. rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
  45. }
  46. rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
  47. }
  48. rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
  49. }
  50. // erasure coding
  51. rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
  52. }
  53. rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
  54. }
  55. rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
  56. }
  57. rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
  58. }
  59. rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
  60. }
  61. rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
  62. }
  63. rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
  64. }
  65. rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
  66. }
  67. rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
  68. }
  69. // tiered storage
  70. rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
  71. }
  72. rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
  73. }
  74. rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
  75. }
  76. rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
  77. }
  78. // <experimental> query
  79. rpc Query (QueryRequest) returns (stream QueriedStripe) {
  80. }
  81. rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
  82. }
  83. }
  84. //////////////////////////////////////////////////
  85. message BatchDeleteRequest {
  86. repeated string file_ids = 1;
  87. bool skip_cookie_check = 2;
  88. }
  89. message BatchDeleteResponse {
  90. repeated DeleteResult results = 1;
  91. }
  92. message DeleteResult {
  93. string file_id = 1;
  94. int32 status = 2;
  95. string error = 3;
  96. uint32 size = 4;
  97. uint32 version = 5;
  98. }
  99. message Empty {
  100. }
  101. message VacuumVolumeCheckRequest {
  102. uint32 volume_id = 1;
  103. }
  104. message VacuumVolumeCheckResponse {
  105. double garbage_ratio = 1;
  106. }
  107. message VacuumVolumeCompactRequest {
  108. uint32 volume_id = 1;
  109. int64 preallocate = 2;
  110. }
  111. message VacuumVolumeCompactResponse {
  112. }
  113. message VacuumVolumeCommitRequest {
  114. uint32 volume_id = 1;
  115. }
  116. message VacuumVolumeCommitResponse {
  117. bool is_read_only = 1;
  118. }
  119. message VacuumVolumeCleanupRequest {
  120. uint32 volume_id = 1;
  121. }
  122. message VacuumVolumeCleanupResponse {
  123. }
  124. message DeleteCollectionRequest {
  125. string collection = 1;
  126. }
  127. message DeleteCollectionResponse {
  128. }
  129. message AllocateVolumeRequest {
  130. uint32 volume_id = 1;
  131. string collection = 2;
  132. int64 preallocate = 3;
  133. string replication = 4;
  134. string ttl = 5;
  135. uint32 memory_map_max_size_mb = 6;
  136. string disk_type = 7;
  137. }
  138. message AllocateVolumeResponse {
  139. }
  140. message VolumeSyncStatusRequest {
  141. uint32 volume_id = 1;
  142. }
  143. message VolumeSyncStatusResponse {
  144. uint32 volume_id = 1;
  145. string collection = 2;
  146. string replication = 4;
  147. string ttl = 5;
  148. uint64 tail_offset = 6;
  149. uint32 compact_revision = 7;
  150. uint64 idx_file_size = 8;
  151. }
  152. message VolumeIncrementalCopyRequest {
  153. uint32 volume_id = 1;
  154. uint64 since_ns = 2;
  155. }
  156. message VolumeIncrementalCopyResponse {
  157. bytes file_content = 1;
  158. }
  159. message VolumeMountRequest {
  160. uint32 volume_id = 1;
  161. }
  162. message VolumeMountResponse {
  163. }
  164. message VolumeUnmountRequest {
  165. uint32 volume_id = 1;
  166. }
  167. message VolumeUnmountResponse {
  168. }
  169. message VolumeDeleteRequest {
  170. uint32 volume_id = 1;
  171. }
  172. message VolumeDeleteResponse {
  173. }
  174. message VolumeMarkReadonlyRequest {
  175. uint32 volume_id = 1;
  176. }
  177. message VolumeMarkReadonlyResponse {
  178. }
  179. message VolumeMarkWritableRequest {
  180. uint32 volume_id = 1;
  181. }
  182. message VolumeMarkWritableResponse {
  183. }
  184. message VolumeConfigureRequest {
  185. uint32 volume_id = 1;
  186. string replication = 2;
  187. }
  188. message VolumeConfigureResponse {
  189. string error = 1;
  190. }
  191. message VolumeStatusRequest {
  192. uint32 volume_id = 1;
  193. }
  194. message VolumeStatusResponse {
  195. bool is_read_only = 1;
  196. }
  197. message VolumeCopyRequest {
  198. uint32 volume_id = 1;
  199. string collection = 2;
  200. string replication = 3;
  201. string ttl = 4;
  202. string source_data_node = 5;
  203. }
  204. message VolumeCopyResponse {
  205. uint64 last_append_at_ns = 1;
  206. }
  207. message CopyFileRequest {
  208. uint32 volume_id = 1;
  209. string ext = 2;
  210. uint32 compaction_revision = 3;
  211. uint64 stop_offset = 4;
  212. string collection = 5;
  213. bool is_ec_volume = 6;
  214. bool ignore_source_file_not_found = 7;
  215. }
  216. message CopyFileResponse {
  217. bytes file_content = 1;
  218. }
  219. message VolumeTailSenderRequest {
  220. uint32 volume_id = 1;
  221. uint64 since_ns = 2;
  222. uint32 idle_timeout_seconds = 3;
  223. }
  224. message VolumeTailSenderResponse {
  225. bytes needle_header = 1;
  226. bytes needle_body = 2;
  227. bool is_last_chunk = 3;
  228. }
  229. message VolumeTailReceiverRequest {
  230. uint32 volume_id = 1;
  231. uint64 since_ns = 2;
  232. uint32 idle_timeout_seconds = 3;
  233. string source_volume_server = 4;
  234. }
  235. message VolumeTailReceiverResponse {
  236. }
  237. message VolumeEcShardsGenerateRequest {
  238. uint32 volume_id = 1;
  239. string collection = 2;
  240. }
  241. message VolumeEcShardsGenerateResponse {
  242. }
  243. message VolumeEcShardsRebuildRequest {
  244. uint32 volume_id = 1;
  245. string collection = 2;
  246. }
  247. message VolumeEcShardsRebuildResponse {
  248. repeated uint32 rebuilt_shard_ids = 1;
  249. }
  250. message VolumeEcShardsCopyRequest {
  251. uint32 volume_id = 1;
  252. string collection = 2;
  253. repeated uint32 shard_ids = 3;
  254. bool copy_ecx_file = 4;
  255. string source_data_node = 5;
  256. bool copy_ecj_file = 6;
  257. bool copy_vif_file = 7;
  258. }
  259. message VolumeEcShardsCopyResponse {
  260. }
  261. message VolumeEcShardsDeleteRequest {
  262. uint32 volume_id = 1;
  263. string collection = 2;
  264. repeated uint32 shard_ids = 3;
  265. }
  266. message VolumeEcShardsDeleteResponse {
  267. }
  268. message VolumeEcShardsMountRequest {
  269. uint32 volume_id = 1;
  270. string collection = 2;
  271. repeated uint32 shard_ids = 3;
  272. }
  273. message VolumeEcShardsMountResponse {
  274. }
  275. message VolumeEcShardsUnmountRequest {
  276. uint32 volume_id = 1;
  277. repeated uint32 shard_ids = 3;
  278. }
  279. message VolumeEcShardsUnmountResponse {
  280. }
  281. message VolumeEcShardReadRequest {
  282. uint32 volume_id = 1;
  283. uint32 shard_id = 2;
  284. int64 offset = 3;
  285. int64 size = 4;
  286. uint64 file_key = 5;
  287. }
  288. message VolumeEcShardReadResponse {
  289. bytes data = 1;
  290. bool is_deleted = 2;
  291. }
  292. message VolumeEcBlobDeleteRequest {
  293. uint32 volume_id = 1;
  294. string collection = 2;
  295. uint64 file_key = 3;
  296. uint32 version = 4;
  297. }
  298. message VolumeEcBlobDeleteResponse {
  299. }
  300. message VolumeEcShardsToVolumeRequest {
  301. uint32 volume_id = 1;
  302. string collection = 2;
  303. }
  304. message VolumeEcShardsToVolumeResponse {
  305. }
  306. message ReadVolumeFileStatusRequest {
  307. uint32 volume_id = 1;
  308. }
  309. message ReadVolumeFileStatusResponse {
  310. uint32 volume_id = 1;
  311. uint64 idx_file_timestamp_seconds = 2;
  312. uint64 idx_file_size = 3;
  313. uint64 dat_file_timestamp_seconds = 4;
  314. uint64 dat_file_size = 5;
  315. uint64 file_count = 6;
  316. uint32 compaction_revision = 7;
  317. string collection = 8;
  318. string disk_type = 9;
  319. }
  320. message DiskStatus {
  321. string dir = 1;
  322. uint64 all = 2;
  323. uint64 used = 3;
  324. uint64 free = 4;
  325. float percent_free = 5;
  326. float percent_used = 6;
  327. string disk_type = 7;
  328. }
  329. message MemStatus {
  330. int32 goroutines = 1;
  331. uint64 all = 2;
  332. uint64 used = 3;
  333. uint64 free = 4;
  334. uint64 self = 5;
  335. uint64 heap = 6;
  336. uint64 stack = 7;
  337. }
  338. // tired storage on volume servers
  339. message RemoteFile {
  340. string backend_type = 1;
  341. string backend_id = 2;
  342. string key = 3;
  343. uint64 offset = 4;
  344. uint64 file_size = 5;
  345. uint64 modified_time = 6;
  346. string extension = 7;
  347. }
  348. message VolumeInfo {
  349. repeated RemoteFile files = 1;
  350. uint32 version = 2;
  351. string replication = 3;
  352. }
  353. message VolumeTierMoveDatToRemoteRequest {
  354. uint32 volume_id = 1;
  355. string collection = 2;
  356. string destination_backend_name = 3;
  357. bool keep_local_dat_file = 4;
  358. }
  359. message VolumeTierMoveDatToRemoteResponse {
  360. int64 processed = 1;
  361. float processedPercentage = 2;
  362. }
  363. message VolumeTierMoveDatFromRemoteRequest {
  364. uint32 volume_id = 1;
  365. string collection = 2;
  366. bool keep_remote_dat_file = 3;
  367. }
  368. message VolumeTierMoveDatFromRemoteResponse {
  369. int64 processed = 1;
  370. float processedPercentage = 2;
  371. }
  372. message VolumeServerStatusRequest {
  373. }
  374. message VolumeServerStatusResponse {
  375. repeated DiskStatus disk_statuses = 1;
  376. MemStatus memory_status = 2;
  377. }
  378. message VolumeServerLeaveRequest {
  379. }
  380. message VolumeServerLeaveResponse {
  381. }
  382. // select on volume servers
  383. message QueryRequest {
  384. repeated string selections = 1;
  385. repeated string from_file_ids = 2;
  386. message Filter {
  387. string field = 1;
  388. string operand = 2;
  389. string value = 3;
  390. }
  391. Filter filter = 3;
  392. message InputSerialization {
  393. // NONE | GZIP | BZIP2
  394. string compression_type = 1;
  395. message CSVInput {
  396. string file_header_info = 1; // Valid values: NONE | USE | IGNORE
  397. string record_delimiter = 2; // Default: \n
  398. string field_delimiter = 3; // Default: ,
  399. string quote_charactoer = 4; // Default: "
  400. string quote_escape_character = 5; // Default: "
  401. string comments = 6; // Default: #
  402. // If true, records might contain record delimiters within quote characters
  403. bool allow_quoted_record_delimiter = 7; // default False.
  404. }
  405. message JSONInput {
  406. string type = 1; // Valid values: DOCUMENT | LINES
  407. }
  408. message ParquetInput {
  409. }
  410. CSVInput csv_input = 2;
  411. JSONInput json_input = 3;
  412. ParquetInput parquet_input = 4;
  413. }
  414. InputSerialization input_serialization = 4;
  415. message OutputSerialization {
  416. message CSVOutput {
  417. string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
  418. string record_delimiter = 2; // Default: \n
  419. string field_delimiter = 3; // Default: ,
  420. string quote_charactoer = 4; // Default: "
  421. string quote_escape_character = 5; // Default: "
  422. }
  423. message JSONOutput {
  424. string record_delimiter = 1;
  425. }
  426. CSVOutput csv_output = 2;
  427. JSONOutput json_output = 3;
  428. }
  429. OutputSerialization output_serialization = 5;
  430. }
  431. message QueriedStripe {
  432. bytes records = 1;
  433. }
  434. message VolumeNeedleStatusRequest {
  435. uint32 volume_id = 1;
  436. uint64 needle_id = 2;
  437. }
  438. message VolumeNeedleStatusResponse {
  439. uint64 needle_id = 1;
  440. uint32 cookie = 2;
  441. uint32 size = 3;
  442. uint64 last_modified = 4;
  443. uint32 crc = 5;
  444. string ttl = 6;
  445. }