Chris Lu
6 years ago
11 changed files with 584 additions and 98 deletions
-
1.gitignore
-
70other/java/client/pom.xml
-
45other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
-
178other/java/client/src/main/proto/filer.proto
-
9other/java/hdfs/pom.xml
-
51other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
-
120other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
2weed/command/filer.go
-
2weed/pb/Makefile
-
9weed/pb/filer.proto
-
175weed/pb/filer_pb/filer.pb.go
@ -0,0 +1,70 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" |
||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<groupId>seaweedfs</groupId> |
||||
|
<artifactId>client</artifactId> |
||||
|
<version>1.0-SNAPSHOT</version> |
||||
|
|
||||
|
<properties> |
||||
|
<protobuf.version>3.5.1</protobuf.version> |
||||
|
<grpc.version>1.16.1</grpc.version> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencies> |
||||
|
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java --> |
||||
|
<dependency> |
||||
|
<groupId>com.google.protobuf</groupId> |
||||
|
<artifactId>protobuf-java</artifactId> |
||||
|
<version>${protobuf.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-netty-shaded</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-protobuf</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-stub</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<extensions> |
||||
|
<extension> |
||||
|
<groupId>kr.motd.maven</groupId> |
||||
|
<artifactId>os-maven-plugin</artifactId> |
||||
|
<version>1.5.0.Final</version> |
||||
|
</extension> |
||||
|
</extensions> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.xolstice.maven.plugins</groupId> |
||||
|
<artifactId>protobuf-maven-plugin</artifactId> |
||||
|
<version>0.5.1</version> |
||||
|
<configuration> |
||||
|
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}-1:exe:${os.detected.classifier}</protocArtifact> |
||||
|
<pluginId>grpc-java</pluginId> |
||||
|
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact> |
||||
|
</configuration> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<goals> |
||||
|
<goal>compile</goal> |
||||
|
<goal>compile-custom</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
|
||||
|
</project> |
||||
@ -0,0 +1,45 @@ |
|||||
|
package seaweedfs.client; |
||||
|
|
||||
|
import io.grpc.ManagedChannel; |
||||
|
import io.grpc.ManagedChannelBuilder; |
||||
|
|
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
import java.util.logging.Logger; |
||||
|
|
||||
|
public class FilerGrpcClient { |
||||
|
|
||||
|
private static final Logger logger = Logger.getLogger(FilerGrpcClient.class.getName()); |
||||
|
|
||||
|
private final ManagedChannel channel; |
||||
|
private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; |
||||
|
private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; |
||||
|
private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub; |
||||
|
|
||||
|
|
||||
|
public FilerGrpcClient(String host, int port) { |
||||
|
this(ManagedChannelBuilder.forAddress(host, port).usePlaintext()); |
||||
|
} |
||||
|
|
||||
|
public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) { |
||||
|
channel = channelBuilder.build(); |
||||
|
blockingStub = SeaweedFilerGrpc.newBlockingStub(channel); |
||||
|
asyncStub = SeaweedFilerGrpc.newStub(channel); |
||||
|
futureStub = SeaweedFilerGrpc.newFutureStub(channel); |
||||
|
} |
||||
|
|
||||
|
public void shutdown() throws InterruptedException { |
||||
|
channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); |
||||
|
} |
||||
|
|
||||
|
public SeaweedFilerGrpc.SeaweedFilerBlockingStub getBlockingStub() { |
||||
|
return blockingStub; |
||||
|
} |
||||
|
|
||||
|
public SeaweedFilerGrpc.SeaweedFilerStub getAsyncStub() { |
||||
|
return asyncStub; |
||||
|
} |
||||
|
|
||||
|
public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() { |
||||
|
return futureStub; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,178 @@ |
|||||
|
syntax = "proto3"; |
||||
|
|
||||
|
package filer_pb; |
||||
|
|
||||
|
option java_package = "seaweedfs.client"; |
||||
|
option java_outer_classname = "FilerProto"; |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
service SeaweedFiler { |
||||
|
|
||||
|
rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) { |
||||
|
} |
||||
|
|
||||
|
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
////////////////////////////////////////////////// |
||||
|
|
||||
|
message LookupDirectoryEntryRequest { |
||||
|
string directory = 1; |
||||
|
string name = 2; |
||||
|
} |
||||
|
|
||||
|
message LookupDirectoryEntryResponse { |
||||
|
Entry entry = 1; |
||||
|
} |
||||
|
|
||||
|
message ListEntriesRequest { |
||||
|
string directory = 1; |
||||
|
string prefix = 2; |
||||
|
string startFromFileName = 3; |
||||
|
bool inclusiveStartFrom = 4; |
||||
|
uint32 limit = 5; |
||||
|
} |
||||
|
|
||||
|
message ListEntriesResponse { |
||||
|
repeated Entry entries = 1; |
||||
|
} |
||||
|
|
||||
|
message Entry { |
||||
|
string name = 1; |
||||
|
bool is_directory = 2; |
||||
|
repeated FileChunk chunks = 3; |
||||
|
FuseAttributes attributes = 4; |
||||
|
map<string, bytes> extended = 5; |
||||
|
} |
||||
|
|
||||
|
message EventNotification { |
||||
|
Entry old_entry = 1; |
||||
|
Entry new_entry = 2; |
||||
|
bool delete_chunks = 3; |
||||
|
} |
||||
|
|
||||
|
message FileChunk { |
||||
|
string file_id = 1; |
||||
|
int64 offset = 2; |
||||
|
uint64 size = 3; |
||||
|
int64 mtime = 4; |
||||
|
string e_tag = 5; |
||||
|
string source_file_id = 6; |
||||
|
} |
||||
|
|
||||
|
message FuseAttributes { |
||||
|
uint64 file_size = 1; |
||||
|
int64 mtime = 2; // unix time in seconds |
||||
|
uint32 file_mode = 3; |
||||
|
uint32 uid = 4; |
||||
|
uint32 gid = 5; |
||||
|
int64 crtime = 6; // unix time in seconds |
||||
|
string mime = 7; |
||||
|
string replication = 8; |
||||
|
string collection = 9; |
||||
|
int32 ttl_sec = 10; |
||||
|
string user_name = 11; // for hdfs |
||||
|
repeated string group_name = 12; // for hdfs |
||||
|
} |
||||
|
|
||||
|
message CreateEntryRequest { |
||||
|
string directory = 1; |
||||
|
Entry entry = 2; |
||||
|
} |
||||
|
|
||||
|
message CreateEntryResponse { |
||||
|
} |
||||
|
|
||||
|
message UpdateEntryRequest { |
||||
|
string directory = 1; |
||||
|
Entry entry = 2; |
||||
|
} |
||||
|
message UpdateEntryResponse { |
||||
|
} |
||||
|
|
||||
|
message DeleteEntryRequest { |
||||
|
string directory = 1; |
||||
|
string name = 2; |
||||
|
bool is_directory = 3; |
||||
|
bool is_delete_data = 4; |
||||
|
bool is_recursive = 5; |
||||
|
} |
||||
|
|
||||
|
message DeleteEntryResponse { |
||||
|
} |
||||
|
|
||||
|
message AssignVolumeRequest { |
||||
|
int32 count = 1; |
||||
|
string collection = 2; |
||||
|
string replication = 3; |
||||
|
int32 ttl_sec = 4; |
||||
|
string data_center = 5; |
||||
|
} |
||||
|
|
||||
|
message AssignVolumeResponse { |
||||
|
string file_id = 1; |
||||
|
string url = 2; |
||||
|
string public_url = 3; |
||||
|
int32 count = 4; |
||||
|
} |
||||
|
|
||||
|
message LookupVolumeRequest { |
||||
|
repeated string volume_ids = 1; |
||||
|
} |
||||
|
|
||||
|
message Locations { |
||||
|
repeated Location locations = 1; |
||||
|
} |
||||
|
|
||||
|
message Location { |
||||
|
string url = 1; |
||||
|
string public_url = 2; |
||||
|
} |
||||
|
message LookupVolumeResponse { |
||||
|
map<string, Locations> locations_map = 1; |
||||
|
} |
||||
|
|
||||
|
message DeleteCollectionRequest { |
||||
|
string collection = 1; |
||||
|
} |
||||
|
|
||||
|
message DeleteCollectionResponse { |
||||
|
} |
||||
|
|
||||
|
message StatisticsRequest { |
||||
|
string replication = 1; |
||||
|
string collection = 2; |
||||
|
string ttl = 3; |
||||
|
} |
||||
|
message StatisticsResponse { |
||||
|
string replication = 1; |
||||
|
string collection = 2; |
||||
|
string ttl = 3; |
||||
|
uint64 total_size = 4; |
||||
|
uint64 used_size = 5; |
||||
|
uint64 file_count = 6; |
||||
|
} |
||||
@ -0,0 +1,120 @@ |
|||||
|
package seaweed.hdfs; |
||||
|
|
||||
|
import org.apache.hadoop.fs.FileStatus; |
||||
|
import org.apache.hadoop.fs.Path; |
||||
|
import org.apache.hadoop.fs.permission.FsPermission; |
||||
|
import org.apache.hadoop.security.UserGroupInformation; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import seaweedfs.client.FilerGrpcClient; |
||||
|
import seaweedfs.client.FilerProto; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Arrays; |
||||
|
import java.util.List; |
||||
|
|
||||
|
public class SeaweedFileSystemStore { |
||||
|
|
||||
|
private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class); |
||||
|
|
||||
|
private FilerGrpcClient filerGrpcClient; |
||||
|
|
||||
|
public SeaweedFileSystemStore(String host, int port) { |
||||
|
filerGrpcClient = new FilerGrpcClient(host, port); |
||||
|
} |
||||
|
|
||||
|
public boolean createDirectory(final Path path, UserGroupInformation currentUser, |
||||
|
final FsPermission permission, final FsPermission umask) { |
||||
|
|
||||
|
LOG.debug("createDirectory path: {} permission: {} umask: {}", |
||||
|
path, |
||||
|
permission, |
||||
|
umask); |
||||
|
|
||||
|
long now = System.currentTimeMillis() / 1000L; |
||||
|
|
||||
|
FilerProto.CreateEntryRequest.Builder request = FilerProto.CreateEntryRequest.newBuilder() |
||||
|
.setDirectory(path.getParent().toUri().getPath()) |
||||
|
.setEntry(FilerProto.Entry.newBuilder() |
||||
|
.setName(path.getName()) |
||||
|
.setIsDirectory(true) |
||||
|
.setAttributes(FilerProto.FuseAttributes.newBuilder() |
||||
|
.setMtime(now) |
||||
|
.setCrtime(now) |
||||
|
.setFileMode(permission.toShort()) |
||||
|
.setUserName(currentUser.getUserName()) |
||||
|
.addAllGroupName(Arrays.asList(currentUser.getGroupNames()))) |
||||
|
); |
||||
|
|
||||
|
FilerProto.CreateEntryResponse response = filerGrpcClient.getBlockingStub().createEntry(request.build()); |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
public FileStatus[] listEntries(final Path path) { |
||||
|
LOG.debug("listEntries path: {}", path); |
||||
|
|
||||
|
List<FileStatus> fileStatuses = new ArrayList<FileStatus>(); |
||||
|
|
||||
|
FilerProto.ListEntriesResponse response = |
||||
|
filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() |
||||
|
.setDirectory(path.toUri().getPath()) |
||||
|
.setLimit(100000) |
||||
|
.build()); |
||||
|
|
||||
|
for (FilerProto.Entry entry : response.getEntriesList()) { |
||||
|
|
||||
|
FileStatus fileStatus = getFileStatus(new Path(path, entry.getName()), entry); |
||||
|
|
||||
|
fileStatuses.add(fileStatus); |
||||
|
} |
||||
|
return fileStatuses.toArray(new FileStatus[0]); |
||||
|
} |
||||
|
|
||||
|
public FileStatus getFileStatus(final Path path) { |
||||
|
LOG.debug("getFileStatus path: {}", path); |
||||
|
|
||||
|
FilerProto.LookupDirectoryEntryResponse response = |
||||
|
filerGrpcClient.getBlockingStub().lookupDirectoryEntry(FilerProto.LookupDirectoryEntryRequest.newBuilder() |
||||
|
.setDirectory(path.getParent().toUri().getPath()) |
||||
|
.setName(path.getName()) |
||||
|
.build()); |
||||
|
|
||||
|
FilerProto.Entry entry = response.getEntry(); |
||||
|
FileStatus fileStatus = getFileStatus(path, entry); |
||||
|
return fileStatus; |
||||
|
} |
||||
|
|
||||
|
public boolean deleteEntries(final Path path, boolean isDirectroy, boolean recursive) { |
||||
|
LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}", |
||||
|
path, |
||||
|
String.valueOf(isDirectroy), |
||||
|
String.valueOf(recursive)); |
||||
|
|
||||
|
FilerProto.DeleteEntryResponse response = |
||||
|
filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder() |
||||
|
.setDirectory(path.getParent().toUri().getPath()) |
||||
|
.setName(path.getName()) |
||||
|
.setIsDirectory(isDirectroy) |
||||
|
.setIsDeleteData(true) |
||||
|
.build()); |
||||
|
|
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private FileStatus getFileStatus(Path path, FilerProto.Entry entry) { |
||||
|
FilerProto.FuseAttributes attributes = entry.getAttributes(); |
||||
|
long length = attributes.getFileSize(); |
||||
|
boolean isDir = entry.getIsDirectory(); |
||||
|
int block_replication = 1; |
||||
|
int blocksize = 512; |
||||
|
long modification_time = attributes.getMtime(); |
||||
|
long access_time = 0; |
||||
|
FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode()); |
||||
|
String owner = attributes.getUserName(); |
||||
|
String group = attributes.getGroupNameCount() > 0 ? attributes.getGroupName(0) : ""; |
||||
|
return new FileStatus(length, isDir, block_replication, blocksize, |
||||
|
modification_time, access_time, permission, owner, group, null, path); |
||||
|
} |
||||
|
|
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue