mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
HCFS: 1.2.8 fix hbase related bugs
1. SeaweedFileSystem.listStatus need to work with file also 2. SeaweedRead readChunkView has wrong len
This commit is contained in:
parent
e5a0787653
commit
8dfaaeabfd
|
@ -5,7 +5,7 @@
|
||||||
|
|
||||||
<groupId>com.github.chrislusf</groupId>
|
<groupId>com.github.chrislusf</groupId>
|
||||||
<artifactId>seaweedfs-client</artifactId>
|
<artifactId>seaweedfs-client</artifactId>
|
||||||
<version>1.2.7</version>
|
<version>1.2.8</version>
|
||||||
|
|
||||||
<parent>
|
<parent>
|
||||||
<groupId>org.sonatype.oss</groupId>
|
<groupId>org.sonatype.oss</groupId>
|
||||||
|
|
139
other/java/client/pom_debug.xml
Normal file
139
other/java/client/pom_debug.xml
Normal file
|
@ -0,0 +1,139 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>com.github.chrislusf</groupId>
|
||||||
|
<artifactId>seaweedfs-client</artifactId>
|
||||||
|
<version>1.2.8</version>
|
||||||
|
|
||||||
|
<parent>
|
||||||
|
<groupId>org.sonatype.oss</groupId>
|
||||||
|
<artifactId>oss-parent</artifactId>
|
||||||
|
<version>9</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<protobuf.version>3.9.1</protobuf.version>
|
||||||
|
<!-- follow https://github.com/grpc/grpc-java -->
|
||||||
|
<grpc.version>1.23.0</grpc.version>
|
||||||
|
<guava.version>28.0-jre</guava.version>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.moandjiezana.toml</groupId>
|
||||||
|
<artifactId>toml4j</artifactId>
|
||||||
|
<version>0.7.2</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.protobuf</groupId>
|
||||||
|
<artifactId>protobuf-java</artifactId>
|
||||||
|
<version>${protobuf.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.guava</groupId>
|
||||||
|
<artifactId>guava</artifactId>
|
||||||
|
<version>${guava.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-netty-shaded</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-protobuf</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.grpc</groupId>
|
||||||
|
<artifactId>grpc-stub</artifactId>
|
||||||
|
<version>${grpc.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>1.7.25</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.httpcomponents</groupId>
|
||||||
|
<artifactId>httpmime</artifactId>
|
||||||
|
<version>4.5.6</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.12</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<extensions>
|
||||||
|
<extension>
|
||||||
|
<groupId>kr.motd.maven</groupId>
|
||||||
|
<artifactId>os-maven-plugin</artifactId>
|
||||||
|
<version>1.6.2</version>
|
||||||
|
</extension>
|
||||||
|
</extensions>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<source>8</source>
|
||||||
|
<target>8</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.xolstice.maven.plugins</groupId>
|
||||||
|
<artifactId>protobuf-maven-plugin</artifactId>
|
||||||
|
<version>0.6.1</version>
|
||||||
|
<configuration>
|
||||||
|
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
|
||||||
|
</protocArtifact>
|
||||||
|
<pluginId>grpc-java</pluginId>
|
||||||
|
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
|
||||||
|
</pluginArtifact>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<goals>
|
||||||
|
<goal>compile</goal>
|
||||||
|
<goal>compile-custom</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-source-plugin</artifactId>
|
||||||
|
<version>2.2.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-sources</id>
|
||||||
|
<goals>
|
||||||
|
<goal>jar-no-fork</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-javadoc-plugin</artifactId>
|
||||||
|
<version>2.9.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-javadocs</id>
|
||||||
|
<goals>
|
||||||
|
<goal>jar</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</project>
|
|
@ -7,6 +7,8 @@ import org.apache.http.client.HttpClient;
|
||||||
import org.apache.http.client.methods.HttpGet;
|
import org.apache.http.client.methods.HttpGet;
|
||||||
import org.apache.http.impl.client.DefaultHttpClient;
|
import org.apache.http.impl.client.DefaultHttpClient;
|
||||||
import org.apache.http.util.EntityUtils;
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -14,7 +16,7 @@ import java.util.*;
|
||||||
|
|
||||||
public class SeaweedRead {
|
public class SeaweedRead {
|
||||||
|
|
||||||
// private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
|
private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
|
||||||
|
|
||||||
static ChunkCache chunkCache = new ChunkCache(1000);
|
static ChunkCache chunkCache = new ChunkCache(1000);
|
||||||
|
|
||||||
|
@ -64,7 +66,9 @@ public class SeaweedRead {
|
||||||
chunkData = doFetchFullChunkData(chunkView, locations);
|
chunkData = doFetchFullChunkData(chunkView, locations);
|
||||||
}
|
}
|
||||||
|
|
||||||
int len = (int) (chunkView.logicOffset - position + chunkView.size);
|
int len = (int) chunkView.size;
|
||||||
|
LOG.debug("readChunkView fid:{} chunkData.length:{} chunkView.offset:{} buffer.length:{} startOffset:{} len:{}",
|
||||||
|
chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len);
|
||||||
System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len);
|
System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len);
|
||||||
|
|
||||||
chunkCache.setChunk(chunkView.fileId, chunkData);
|
chunkCache.setChunk(chunkView.fileId, chunkData);
|
||||||
|
|
|
@ -127,7 +127,7 @@
|
||||||
</snapshotRepository>
|
</snapshotRepository>
|
||||||
</distributionManagement>
|
</distributionManagement>
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>1.2.7</seaweedfs.client.version>
|
<seaweedfs.client.version>1.2.8</seaweedfs.client.version>
|
||||||
<hadoop.version>2.9.2</hadoop.version>
|
<hadoop.version>2.9.2</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>1.2.7</seaweedfs.client.version>
|
<seaweedfs.client.version>1.2.8</seaweedfs.client.version>
|
||||||
<hadoop.version>2.9.2</hadoop.version>
|
<hadoop.version>2.9.2</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
|
|
|
@ -64,6 +64,16 @@ public class SeaweedFileSystemStore {
|
||||||
public FileStatus[] listEntries(final Path path) {
|
public FileStatus[] listEntries(final Path path) {
|
||||||
LOG.debug("listEntries path: {}", path);
|
LOG.debug("listEntries path: {}", path);
|
||||||
|
|
||||||
|
FileStatus pathStatus = getFileStatus(path);
|
||||||
|
|
||||||
|
if (pathStatus == null) {
|
||||||
|
return new FileStatus[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!pathStatus.isDirectory()) {
|
||||||
|
return new FileStatus[]{pathStatus};
|
||||||
|
}
|
||||||
|
|
||||||
List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
|
List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
|
||||||
|
|
||||||
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
|
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
|
||||||
|
@ -74,7 +84,9 @@ public class SeaweedFileSystemStore {
|
||||||
|
|
||||||
fileStatuses.add(fileStatus);
|
fileStatuses.add(fileStatus);
|
||||||
}
|
}
|
||||||
|
LOG.debug("listEntries path: {} size {}", fileStatuses, fileStatuses.size());
|
||||||
return fileStatuses.toArray(new FileStatus[0]);
|
return fileStatuses.toArray(new FileStatus[0]);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileStatus getFileStatus(final Path path) {
|
public FileStatus getFileStatus(final Path path) {
|
||||||
|
|
|
@ -127,7 +127,7 @@
|
||||||
</snapshotRepository>
|
</snapshotRepository>
|
||||||
</distributionManagement>
|
</distributionManagement>
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>1.2.7</seaweedfs.client.version>
|
<seaweedfs.client.version>1.2.8</seaweedfs.client.version>
|
||||||
<hadoop.version>3.1.1</hadoop.version>
|
<hadoop.version>3.1.1</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<seaweedfs.client.version>1.2.7</seaweedfs.client.version>
|
<seaweedfs.client.version>1.2.8</seaweedfs.client.version>
|
||||||
<hadoop.version>3.1.1</hadoop.version>
|
<hadoop.version>3.1.1</hadoop.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
|
|
|
@ -64,6 +64,16 @@ public class SeaweedFileSystemStore {
|
||||||
public FileStatus[] listEntries(final Path path) {
|
public FileStatus[] listEntries(final Path path) {
|
||||||
LOG.debug("listEntries path: {}", path);
|
LOG.debug("listEntries path: {}", path);
|
||||||
|
|
||||||
|
FileStatus pathStatus = getFileStatus(path);
|
||||||
|
|
||||||
|
if (pathStatus == null) {
|
||||||
|
return new FileStatus[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!pathStatus.isDirectory()) {
|
||||||
|
return new FileStatus[]{pathStatus};
|
||||||
|
}
|
||||||
|
|
||||||
List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
|
List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
|
||||||
|
|
||||||
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
|
List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
|
||||||
|
@ -74,7 +84,9 @@ public class SeaweedFileSystemStore {
|
||||||
|
|
||||||
fileStatuses.add(fileStatus);
|
fileStatuses.add(fileStatus);
|
||||||
}
|
}
|
||||||
|
LOG.debug("listEntries path: {} size {}", fileStatuses, fileStatuses.size());
|
||||||
return fileStatuses.toArray(new FileStatus[0]);
|
return fileStatuses.toArray(new FileStatus[0]);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public FileStatus getFileStatus(final Path path) {
|
public FileStatus getFileStatus(final Path path) {
|
||||||
|
|
Loading…
Reference in a new issue