Browse Source

avoid range request for gzipped content

pull/800/head
Chris Lu 6 years ago
parent
commit
6946c51430
  1. 2
      other/java/hdfs/pom.xml
  2. 12
      other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedRead.java

2
other/java/hdfs/pom.xml

@ -72,7 +72,7 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId> <artifactId>httpmime</artifactId>
<version>4.5.2</version>
<version>4.5.6</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

12
other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedRead.java

@ -1,6 +1,8 @@
package seaweed.hdfs; package seaweed.hdfs;
import org.apache.hadoop.hdfs.util.ByteBufferOutputStream;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
@ -11,6 +13,8 @@ import seaweedfs.client.FilerGrpcClient;
import seaweedfs.client.FilerProto; import seaweedfs.client.FilerProto;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
@ -52,7 +56,8 @@ public class SeaweedRead {
HttpClient client = HttpClientBuilder.create().build(); HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet( HttpGet request = new HttpGet(
String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId)); String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
request.setHeader("Range",
request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
request.setHeader(HttpHeaders.RANGE,
String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size)); String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
try { try {
@ -60,9 +65,10 @@ public class SeaweedRead {
HttpEntity entity = response.getEntity(); HttpEntity entity = response.getEntity();
int len = (int) (chunkView.logicOffset - position + chunkView.size); int len = (int) (chunkView.logicOffset - position + chunkView.size);
int chunReadCount = entity.getContent().read(buffer, startOffset, len);
OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
entity.writeTo(outputStream);
LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
LOG.debug("* read chunkView:{} startOffset:{} length:{} chunReadCount:{}", chunkView, startOffset, len, chunReadCount);
readCount += len; readCount += len;
startOffset += len; startOffset += len;

Loading…
Cancel
Save