From 6b41c5250bd8119c8fb5485ca9a2f2e42798a91e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 21:33:10 -0700 Subject: [PATCH] Hadoop file system: 1.4.3 added buffered fs input stream --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- .../hdfs/BufferedSeaweedInputStream.java | 49 ------------------- .../java/seaweed/hdfs/SeaweedFileSystem.java | 4 +- .../seaweed/hdfs/SeaweedFileSystemStore.java | 5 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- .../hdfs/BufferedSeaweedInputStream.java | 49 ------------------- .../java/seaweed/hdfs/SeaweedFileSystem.java | 4 +- .../seaweed/hdfs/SeaweedFileSystemStore.java | 5 +- 13 files changed, 17 insertions(+), 113 deletions(-) delete mode 100644 other/java/hdfs2/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java delete mode 100644 other/java/hdfs3/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index b2e88935c..57233e86d 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.1 + 1.4.3 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index b2e88935c..57233e86d 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.1 + 1.4.3 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index d2d4cbb5d..67bab76ea 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.4.1 + 1.4.3 org.sonatype.oss diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 9d0c501ae..8df1a6356 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -127,7 +127,7 @@ - 1.4.1 + 1.4.3 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 25a968c07..3406e12cb 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.4.1 + 1.4.3 2.9.2 diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java deleted file mode 100644 index 0bee6e43f..000000000 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java +++ /dev/null @@ -1,49 +0,0 @@ -package seaweed.hdfs; - -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; - -import java.io.BufferedInputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; - -class BufferedSeaweedInputStream extends FilterInputStream implements Seekable, PositionedReadable { - - SeaweedInputStream t; - - protected BufferedSeaweedInputStream(InputStream in, int bufferSize) { - super(new BufferedInputStream(in, bufferSize)); - t = (SeaweedInputStream)in; - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) throws IOException { - return this.t.read(position,buffer,offset,length); - } - - @Override - public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { - this.t.readFully(position,buffer,offset,length); - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - this.t.readFully(position,buffer); - } - - @Override - public void seek(long pos) throws IOException { - this.t.seek(pos); - } - - @Override - public long getPos() throws IOException { - return this.t.getPos(); - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - return this.t.seekToNewSource(targetPos); - } -} diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 585fa39a3..fd8877806 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -75,8 +75,8 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize); - return new FSDataInputStream(new BufferedSeaweedInputStream(inputStream, 16 * 1024 * 1024)); + FSInputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize); + return new FSDataInputStream(new BufferedFSInputStream(inputStream, 16 * 1024 * 1024)); } catch (Exception ex) { LOG.warn("open path: {} bufferSize:{}", path, bufferSize, ex); return null; diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index d9c6d6f0d..0db6a1f49 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -1,5 +1,6 @@ package seaweed.hdfs; +import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -207,8 +208,8 @@ public class SeaweedFileSystemStore { } - public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics, - int bufferSize) throws IOException { + public FSInputStream openFileForRead(final Path path, FileSystem.Statistics statistics, + int bufferSize) throws IOException { LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize); diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 0b01074c3..5b13f7d02 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -127,7 +127,7 @@ - 1.4.1 + 1.4.3 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 025843d3c..e5209105d 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.4.1 + 1.4.3 3.1.1 diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java deleted file mode 100644 index 0bee6e43f..000000000 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/BufferedSeaweedInputStream.java +++ /dev/null @@ -1,49 +0,0 @@ -package seaweed.hdfs; - -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; - -import java.io.BufferedInputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; - -class BufferedSeaweedInputStream extends FilterInputStream implements Seekable, PositionedReadable { - - SeaweedInputStream t; - - protected BufferedSeaweedInputStream(InputStream in, int bufferSize) { - super(new BufferedInputStream(in, bufferSize)); - t = (SeaweedInputStream)in; - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) throws IOException { - return this.t.read(position,buffer,offset,length); - } - - @Override - public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { - this.t.readFully(position,buffer,offset,length); - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - this.t.readFully(position,buffer); - } - - @Override - public void seek(long pos) throws IOException { - this.t.seek(pos); - } - - @Override - public long getPos() throws IOException { - return this.t.getPos(); - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - return this.t.seekToNewSource(targetPos); - } -} diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 585fa39a3..fd8877806 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -75,8 +75,8 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize); - return new FSDataInputStream(new BufferedSeaweedInputStream(inputStream, 16 * 1024 * 1024)); + FSInputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize); + return new FSDataInputStream(new BufferedFSInputStream(inputStream, 16 * 1024 * 1024)); } catch (Exception ex) { LOG.warn("open path: {} bufferSize:{}", path, bufferSize, ex); return null; diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index d9c6d6f0d..0db6a1f49 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -1,5 +1,6 @@ package seaweed.hdfs; +import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -207,8 +208,8 @@ public class SeaweedFileSystemStore { } - public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics, - int bufferSize) throws IOException { + public FSInputStream openFileForRead(final Path path, FileSystem.Statistics statistics, + int bufferSize) throws IOException { LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);