1.3.0

* add option to include last entry *
5 years ago · 4bd8f3281e
10 changed files with 43 additions and 25 deletions
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.2.9</version>
+    <version>1.3.0</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
--- a/other/java/client/pom_debug.xml
+++ b/other/java/client/pom_debug.xml
@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.2.9</version>
+    <version>1.3.0</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
--- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
@ -156,7 +156,7 @@ public class FilerClient {
        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
        String lastFileName = "";
        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
-            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024, false);
            if (t == null) {
                break;
            }
@ -173,11 +173,12 @@ public class FilerClient {
        return results;
    }

-    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit, boolean includeLastEntry) {
        Iterator<FilerProto.ListEntriesResponse> iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
                .setDirectory(path)
                .setPrefix(entryPrefix)
                .setStartFromFileName(lastEntryName)
+                .setInclusiveStartFrom(includeLastEntry)
                .setLimit(limit)
                .build());
        List<FilerProto.Entry> entries = new ArrayList<>();
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
@ -4,9 +4,9 @@ import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@ -79,7 +79,6 @@ public class SeaweedRead {

    private static byte[] doFetchFullChunkData(ChunkView chunkView, FilerProto.Locations locations) throws IOException {

-        HttpClient client = HttpClientBuilder.create().build();
        HttpGet request = new HttpGet(
                String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));

@ -87,17 +86,15 @@ public class SeaweedRead {

        byte[] data = null;

+        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(request);
+
        try {
-            HttpResponse response = client.execute(request);
            HttpEntity entity = response.getEntity();

            data = EntityUtils.toByteArray(entity);

        } finally {
-            if (client instanceof Closeable) {
-                Closeable t = (Closeable) client;
-                t.close();
-            }
+            response.close();
        }

        if (chunkView.isCompressed) {
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
@ -0,0 +1,24 @@
+package seaweedfs.client;
+
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+
+public class SeaweedUtil {
+
+    static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
+
+    static {
+        // Increase max total connection to 200
+        cm.setMaxTotal(200);
+        // Increase default max connection per route to 20
+        cm.setDefaultMaxPerRoute(20);
+    }
+
+    public static CloseableHttpClient getClosableHttpClient() {
+        return HttpClientBuilder.create()
+                .setConnectionManager(cm)
+                .build();
+    }
+}
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
@ -3,10 +3,10 @@ package seaweedfs.client;
 import com.google.protobuf.ByteString;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.mime.HttpMultipartMode;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.impl.client.HttpClientBuilder;

 import java.io.ByteArrayInputStream;
 import java.io.Closeable;
@ -16,7 +16,7 @@ import java.security.SecureRandom;

 public class SeaweedWrite {

-    private static SecureRandom random = new SecureRandom();
+    private static final SecureRandom random = new SecureRandom();

    public static void writeData(FilerProto.Entry.Builder entry,
                                 final String replication,
@ -63,7 +63,7 @@ public class SeaweedWrite {

    public static void writeMeta(final FilerGrpcClient filerGrpcClient,
                                 final String parentDirectory, final FilerProto.Entry.Builder entry) {
-        synchronized (entry){
+        synchronized (entry) {
            filerGrpcClient.getBlockingStub().createEntry(
                    FilerProto.CreateEntryRequest.newBuilder()
                            .setDirectory(parentDirectory)
@ -79,8 +79,6 @@ public class SeaweedWrite {
                                          final long bytesOffset, final long bytesLength,
                                          byte[] cipherKey) throws IOException {

-        HttpClient client = HttpClientBuilder.create().build();
-
        InputStream inputStream = null;
        if (cipherKey == null || cipherKey.length == 0) {
            inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
@ -103,8 +101,9 @@ public class SeaweedWrite {
                .addBinaryBody("upload", inputStream)
                .build());

+        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
+
        try {
-            HttpResponse response = client.execute(post);

            String etag = response.getLastHeader("ETag").getValue();

@ -114,10 +113,7 @@ public class SeaweedWrite {

            return etag;
        } finally {
-            if (client instanceof Closeable) {
-                Closeable t = (Closeable) client;
-                t.close();
-            }
+            response.close();
        }

    }
--- a/other/java/hdfs2/dependency-reduced-pom.xml
+++ b/other/java/hdfs2/dependency-reduced-pom.xml
@ -127,7 +127,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.2.9</seaweedfs.client.version>
+    <seaweedfs.client.version>1.3.0</seaweedfs.client.version>
    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
--- a/other/java/hdfs2/pom.xml
+++ b/other/java/hdfs2/pom.xml
@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.2.9</seaweedfs.client.version>
+        <seaweedfs.client.version>1.3.0</seaweedfs.client.version>
        <hadoop.version>2.9.2</hadoop.version>
    </properties>

--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@ -127,7 +127,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.2.9</seaweedfs.client.version>
+    <seaweedfs.client.version>1.3.0</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.2.9</seaweedfs.client.version>
+        <seaweedfs.client.version>1.3.0</seaweedfs.client.version>
        <hadoop.version>3.1.1</hadoop.version>
    </properties>