seaweedfs/weed/pb/filer.proto

syntax = "proto3";

package filer_pb;

option go_package = "github.com/chrislusf/seaweedfs/weed/pb/filer_pb";
option java_package = "seaweedfs.client";
option java_outer_classname = "FilerProto";

//////////////////////////////////////////////////

service SeaweedFiler {

    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
    }

    rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
    }

    rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
    }

    rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
    }

    rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
    }

    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
    }

    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
    }

    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
    }

    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
    }

    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
    }

    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
    }

    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
    }

    rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
    }

    rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
    }

    rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
    }

    rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
    }

    rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) {
    }

    rpc KvGet (KvGetRequest) returns (KvGetResponse) {
    }

    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
    }
}

//////////////////////////////////////////////////

message LookupDirectoryEntryRequest {
    string directory = 1;
    string name = 2;
}

message LookupDirectoryEntryResponse {
    Entry entry = 1;
}

message ListEntriesRequest {
    string directory = 1;
    string prefix = 2;
    string startFromFileName = 3;
    bool inclusiveStartFrom = 4;
    uint32 limit = 5;
}

message ListEntriesResponse {
    Entry entry = 1;
}

message RemoteEntry {
    string storage_name = 1;
    int64 last_local_sync_ts_ns = 2;
    string remote_e_tag = 3;
    int64 remote_mtime = 4;
    int64 remote_size = 5;
}
message Entry {
    string name = 1;
    bool is_directory = 2;
    repeated FileChunk chunks = 3;
    FuseAttributes attributes = 4;
    map<string, bytes> extended = 5;
    bytes hard_link_id = 7;
    int32 hard_link_counter = 8; // only exists in hard link meta data
    bytes content = 9; // if not empty, the file content

    RemoteEntry remote_entry = 10;
}

message FullEntry {
    string dir = 1;
    Entry entry = 2;
}

message EventNotification {
    Entry old_entry = 1;
    Entry new_entry = 2;
    bool delete_chunks = 3;
    string new_parent_path = 4;
    bool is_from_other_cluster = 5;
    repeated int32 signatures = 6;
}

message FileChunk {
    string file_id = 1; // to be deprecated
    int64 offset = 2;
    uint64 size = 3;
    int64 mtime = 4;
    string e_tag = 5;
    string source_file_id = 6; // to be deprecated
    FileId fid = 7;
    FileId source_fid = 8;
    bytes cipher_key = 9;
    bool is_compressed = 10;
    bool is_chunk_manifest = 11; // content is a list of FileChunks
}

message FileChunkManifest {
    repeated FileChunk chunks = 1;
}

message FileId {
    uint32 volume_id = 1;
    uint64 file_key = 2;
    fixed32 cookie = 3;
}

message FuseAttributes {
    uint64 file_size = 1;
    int64 mtime = 2; // unix time in seconds
    uint32 file_mode = 3;
    uint32 uid = 4;
    uint32 gid = 5;
    int64 crtime = 6; // unix time in seconds
    string mime = 7;
    string replication = 8;
    string collection = 9;
    int32 ttl_sec = 10;
    string user_name = 11; // for hdfs
    repeated string group_name = 12; // for hdfs
    string symlink_target = 13;
    bytes md5 = 14;
    string disk_type = 15;
}

message CreateEntryRequest {
    string directory = 1;
    Entry entry = 2;
    bool o_excl = 3;
    bool is_from_other_cluster = 4;
    repeated int32 signatures = 5;
}

message CreateEntryResponse {
    string error = 1;
}

message UpdateEntryRequest {
    string directory = 1;
    Entry entry = 2;
    bool is_from_other_cluster = 3;
    repeated int32 signatures = 4;
}
message UpdateEntryResponse {
}

message AppendToEntryRequest {
    string directory = 1;
    string entry_name = 2;
    repeated FileChunk chunks = 3;
}
message AppendToEntryResponse {
}

message DeleteEntryRequest {
    string directory = 1;
    string name = 2;
    // bool is_directory = 3;
    bool is_delete_data = 4;
    bool is_recursive = 5;
    bool ignore_recursive_error = 6;
    bool is_from_other_cluster = 7;
    repeated int32 signatures = 8;
}

message DeleteEntryResponse {
    string error = 1;
}

message AtomicRenameEntryRequest {
    string old_directory = 1;
    string old_name = 2;
    string new_directory = 3;
    string new_name = 4;
    repeated int32 signatures = 5;
}

message AtomicRenameEntryResponse {
}

message AssignVolumeRequest {
    int32 count = 1;
    string collection = 2;
    string replication = 3;
    int32 ttl_sec = 4;
    string data_center = 5;
    string path = 6;
    string rack = 7;
    string disk_type = 8;
}

message AssignVolumeResponse {
    string file_id = 1;
    string url = 2;
    string public_url = 3;
    int32 count = 4;
    string auth = 5;
    string collection = 6;
    string replication = 7;
    string error = 8;
}

message LookupVolumeRequest {
    repeated string volume_ids = 1;
}

message Locations {
    repeated Location locations = 1;
}

message Location {
    string url = 1;
    string public_url = 2;
}
message LookupVolumeResponse {
    map<string, Locations> locations_map = 1;
}

message Collection {
    string name = 1;
}
message CollectionListRequest {
    bool include_normal_volumes = 1;
    bool include_ec_volumes = 2;
}
message CollectionListResponse {
    repeated Collection collections = 1;
}
message DeleteCollectionRequest {
    string collection = 1;
}

message DeleteCollectionResponse {
}

message StatisticsRequest {
    string replication = 1;
    string collection = 2;
    string ttl = 3;
    string disk_type = 4;
}
message StatisticsResponse {
    uint64 total_size = 4;
    uint64 used_size = 5;
    uint64 file_count = 6;
}

message GetFilerConfigurationRequest {
}
message GetFilerConfigurationResponse {
    repeated string masters = 1;
    string replication = 2;
    string collection = 3;
    uint32 max_mb = 4;
    string dir_buckets = 5;
    bool cipher = 7;
    int32 signature = 8;
    string metrics_address = 9;
    int32 metrics_interval_sec = 10;
    string version = 11;
}

message SubscribeMetadataRequest {
    string client_name = 1;
    string path_prefix = 2;
    int64 since_ns = 3;
    int32 signature = 4;
}
message SubscribeMetadataResponse {
    string directory = 1;
    EventNotification event_notification = 2;
    int64 ts_ns = 3;
}

message LogEntry {
    int64 ts_ns = 1;
    int32 partition_key_hash = 2;
    bytes data = 3;
}

message KeepConnectedRequest {
    string name = 1;
    uint32 grpc_port = 2;
    repeated string resources = 3;
}
message KeepConnectedResponse {
}

message LocateBrokerRequest {
    string resource = 1;
}
message LocateBrokerResponse {
    bool found = 1;
    // if found, send the exact address
    // if not found, send the full list of existing brokers
    message Resource {
        string grpc_addresses = 1;
        int32 resource_count = 2;
    }
    repeated Resource resources = 2;
}

/////////////////////////
// Key-Value operations
/////////////////////////
message KvGetRequest {
    bytes key = 1;
}
message KvGetResponse {
    bytes value = 1;
    string error = 2;
}
message KvPutRequest {
    bytes key = 1;
    bytes value = 2;
}
message KvPutResponse {
    string error = 1;
}

/////////////////////////
// path-based configurations
/////////////////////////
message FilerConf {
    int32 version = 1;
    message PathConf {
        string location_prefix = 1;
        string collection = 2;
        string replication = 3;
        string ttl = 4;
        string disk_type = 5;
        bool fsync = 6;
        uint32 volume_growth_count = 7;
        bool read_only = 8;
    }
    repeated PathConf locations = 2;
}

/////////////////////////
// Remote Storage related
/////////////////////////
message RemoteConf {
    string type = 1;
    string name = 2;
    string s3_access_key = 4;
    string s3_secret_key = 5;
    string s3_region = 6;
    string s3_endpoint = 7;
    string s3_storage_class = 8;
    bool s3_force_path_style = 9;
    string gcs_google_application_credentials = 10;
}

message RemoteStorageMapping {
    map<string,RemoteStorageLocation> mappings = 1;
}
message RemoteStorageLocation {
    string name = 1;
    string bucket = 2;
    string path = 3;
}
message DownloadToLocalRequest {
    string directory = 1;
    string name = 2;
}
message DownloadToLocalResponse {
    Entry entry = 1;
}