gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

avoid range request for gzipped content

Browse source
This commit is contained in:
Chris Lu 2018-12-07 01:23:30 -08:00
parent 6b39effe7f
commit 6946c51430
2 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
other/java/hdfs/pom.xml
View file

@ -72,7 +72,7 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId> <artifactId>httpmime</artifactId>
<version>4.5.2</version> <version>4.5.6</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

12
other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedRead.java
View file

@ -1,6 +1,8 @@
package seaweed.hdfs; package seaweed.hdfs;
import org.apache.hadoop.hdfs.util.ByteBufferOutputStream;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
@ -11,6 +13,8 @@ import seaweedfs.client.FilerGrpcClient;
import seaweedfs.client.FilerProto; import seaweedfs.client.FilerProto;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
@ -52,7 +56,8 @@ public class SeaweedRead {
HttpClient client = HttpClientBuilder.create().build(); HttpClient client = HttpClientBuilder.create().build();
HttpGet request = new HttpGet( HttpGet request = new HttpGet(
String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId)); String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
request.setHeader("Range", request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
request.setHeader(HttpHeaders.RANGE,
String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size)); String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
try { try {
@ -60,9 +65,10 @@ public class SeaweedRead {
HttpEntity entity = response.getEntity(); HttpEntity entity = response.getEntity();
int len = (int) (chunkView.logicOffset - position + chunkView.size); int len = (int) (chunkView.logicOffset - position + chunkView.size);
int chunReadCount = entity.getContent().read(buffer, startOffset, len); OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
entity.writeTo(outputStream);
LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
LOG.debug("* read chunkView:{} startOffset:{} length:{} chunReadCount:{}", chunkView, startOffset, len, chunReadCount);
readCount += len; readCount += len;
startOffset += len; startOffset += len;