Java: 1.6.7 Support Mounted Remote Storage

This commit is contained in:
Chris Lu 2021-08-09 22:46:12 -07:00
parent 69655ba8e5
commit e9128e75d0
10 changed files with 45 additions and 13 deletions

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.6</version> <version>1.6.7</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.6</version> <version>1.6.7</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.6</version> <version>1.6.7</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -0,0 +1,23 @@
package seaweedfs.client;
import java.io.IOException;
public class RemoteUtil {
public static boolean isInRemoteOnly(FilerProto.Entry entry) {
if (entry.getChunksList() == null || entry.getChunksList().isEmpty()) {
return entry.getRemoteEntry() != null && entry.getRemoteEntry().getRemoteSize() > 0;
}
return false;
}
public static FilerProto.Entry downloadRemoteEntry(FilerClient filerClient, String fullpath, FilerProto.Entry entry) throws IOException {
String dir = SeaweedOutputStream.getParentDirectory(fullpath);
String name = SeaweedOutputStream.getFileName(fullpath);
final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
.downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
.setDirectory(dir).setName(name).build());
return downloadToLocalResponse.getEntry();
}
}

View file

@ -19,9 +19,9 @@ public class SeaweedInputStream extends InputStream {
private final FilerClient filerClient; private final FilerClient filerClient;
private final String path; private final String path;
private final FilerProto.Entry entry;
private final List<SeaweedRead.VisibleInterval> visibleIntervalList; private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
private final long contentLength; private final long contentLength;
private FilerProto.Entry entry;
private long position = 0; // cursor of the file private long position = 0; // cursor of the file
@ -35,10 +35,14 @@ public class SeaweedInputStream extends InputStream {
this.entry = filerClient.lookupEntry( this.entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(fullpath), SeaweedOutputStream.getParentDirectory(fullpath),
SeaweedOutputStream.getFileName(fullpath)); SeaweedOutputStream.getFileName(fullpath));
if(entry == null){ if (entry == null) {
throw new FileNotFoundException(); throw new FileNotFoundException();
} }
if (RemoteUtil.isInRemoteOnly(entry)) {
entry = RemoteUtil.downloadRemoteEntry(filerClient, fullpath, entry);
}
this.contentLength = SeaweedRead.fileSize(entry); this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList()); this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@ -54,6 +58,11 @@ public class SeaweedInputStream extends InputStream {
this.filerClient = filerClient; this.filerClient = filerClient;
this.path = path; this.path = path;
this.entry = entry; this.entry = entry;
if (RemoteUtil.isInRemoteOnly(entry)) {
this.entry = RemoteUtil.downloadRemoteEntry(filerClient, path, entry);
}
this.contentLength = SeaweedRead.fileSize(entry); this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList()); this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@ -111,8 +120,8 @@ public class SeaweedInputStream extends InputStream {
long bytesRead = 0; long bytesRead = 0;
int len = buf.remaining(); int len = buf.remaining();
int start = (int) this.position; int start = (int) this.position;
if (start+len <= entry.getContent().size()) { if (start + len <= entry.getContent().size()) {
entry.getContent().substring(start, start+len).copyTo(buf); entry.getContent().substring(start, start + len).copyTo(buf);
} else { } else {
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry)); bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
} }

View file

@ -11,13 +11,13 @@
<dependency> <dependency>
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.6</version> <version>1.6.7</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId> <artifactId>seaweedfs-hadoop2-client</artifactId>
<version>1.6.6</version> <version>1.6.7</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>

View file

@ -301,7 +301,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.6.6</seaweedfs.client.version> <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.6.6</seaweedfs.client.version> <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>

View file

@ -309,7 +309,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.6.6</seaweedfs.client.version> <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.6.6</seaweedfs.client.version> <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>