gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6 from chrislusf/master

Browse source 
sync
This commit is contained in:
hilimd 2020-07-30 09:46:44 +08:00 committed by GitHub
parent 92c32f9d46 5080bc1d69
commit 4a88cca543
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 35 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Makefile
View file

@ -40,7 +40,7 @@ linux: deps
mkdir -p linux
GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
##### LINUX BUILDS #####
5_byte_linux_build:
@ -55,6 +55,14 @@ release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_buil
$(call build_large,windows,amd64,.exe)
$(call zip_large,windows,amd64,.exe)
5_byte_arm_build: $(sources)
$(call build_large,linux,arm,)
$(call tar_large,linux,arm)
5_byte_arm64_build: $(sources)
$(call build_large,linux,arm64,)
$(call tar_large,linux,arm64)
linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
build/linux_386.tar.gz: $(sources)

17
other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java
View file

@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class FileChunkManifest {
@ -51,13 +50,17 @@ public class FileChunkManifest {
private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException {
FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
String vid = "" + chunk.getFid().getVolumeId();
lookupRequest.addVolumeIds(vid);
FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
.getBlockingStub().lookupVolume(lookupRequest.build());
Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
FilerProto.Locations locations = vid2Locations.get(vid);
FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid);
if (locations == null) {
FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
lookupRequest.addVolumeIds(vid);
FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
.getBlockingStub().lookupVolume(lookupRequest.build());
locations = lookupResponse.getLocationsMapMap().get(vid);
filerGrpcClient.vidLocations.put(vid, locations);
LOG.debug("fetchChunk vid:{} locations:{}", vid, locations);
}
SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView(
FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId()

3
other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
View file

@ -9,6 +9,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.util.Map;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
public class FilerGrpcClient {
@ -24,6 +26,7 @@ public class FilerGrpcClient {
}
}
public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
private final ManagedChannel channel;
private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;

7
other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
View file

@ -15,7 +15,7 @@ public class SeaweedRead {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
static ChunkCache chunkCache = new ChunkCache(0);
static ChunkCache chunkCache = new ChunkCache(4);
// returns bytesRead
public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@ -62,6 +62,7 @@ public class SeaweedRead {
if (chunkData == null) {
chunkData = doFetchFullChunkData(chunkView, locations);
chunkCache.setChunk(chunkView.fileId, chunkData);
}
int len = (int) chunkView.size;
@ -69,8 +70,6 @@ public class SeaweedRead {
chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len);
System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len);
chunkCache.setChunk(chunkView.fileId, chunkData);
return len;
}
@ -109,6 +108,8 @@ public class SeaweedRead {
}
}
LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length);
return data;
}

3
other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
View file

@ -61,9 +61,6 @@ public class SeaweedWrite {
String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey);
// cache fileId ~ bytes
SeaweedRead.chunkCache.setChunk(fileId, bytes);
LOG.debug("write file chunk {} size {}", targetUrl, bytesLength);
return FilerProto.FileChunk.newBuilder()

2
weed/command/mount.go
View file

@ -9,7 +9,6 @@ type MountOptions struct {
filerMountRootPath *string
dir *string
dirAutoCreate *bool
dirListCacheLimit *int64
collection *string
replication *string
ttlSec *int
@ -35,7 +34,6 @@ func init() {
mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing")
mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")

1
weed/command/mount_std.go
View file

@ -165,7 +165,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
CacheDir: *option.cacheDir,
CacheSizeMB: *option.cacheSizeMB,
DataCenter: *option.dataCenter,
DirListCacheLimit: *option.dirListCacheLimit,
EntryCacheTtl: 3 * time.Second,
MountUid: uid,
MountGid: gid,

1
weed/command/s3.go
View file

@ -151,6 +151,7 @@ func (s3opt *S3Options) startS3Server() bool {
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filer: *s3opt.filer,
Port: *s3opt.port,
FilerGrpcAddress: filerGrpcAddress,
Config: *s3opt.config,
DomainName: *s3opt.domainName,

3
weed/filesys/dir.go
View file

@ -3,6 +3,7 @@ package filesys
import (
"bytes"
"context"
"math"
"os"
"strings"
"time"
@ -277,7 +278,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
dirPath := util.FullPath(dir.FullPath())
meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath)
listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(dir.wfs.option.DirListCacheLimit))
listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32))
if listErr != nil {
glog.Errorf("list meta cache: %v", listErr)
return nil, fuse.EIO

1
weed/filesys/wfs.go
View file

@ -34,7 +34,6 @@ type Option struct {
CacheDir string
CacheSizeMB int64
DataCenter string
DirListCacheLimit int64
EntryCacheTtl time.Duration
Umask os.FileMode

7
weed/s3api/s3api_server.go
View file

@ -1,6 +1,7 @@
package s3api
import (
"fmt"
"net/http"
"github.com/gorilla/mux"
@ -9,6 +10,7 @@ import (
type S3ApiServerOption struct {
Filer string
Port int
FilerGrpcAddress string
Config string
DomainName string
@ -37,7 +39,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
apiRouter := router.PathPrefix("/").Subrouter()
var routers []*mux.Router
if s3a.option.DomainName != "" {
routers = append(routers, apiRouter.Host("{bucket:.+}."+s3a.option.DomainName).Subrouter())
routers = append(routers, apiRouter.Host(
fmt.Sprintf("%s.%s:%d", "{bucket:.+}", s3a.option.DomainName, s3a.option.Port)).Subrouter())
routers = append(routers, apiRouter.Host(
fmt.Sprintf("%s.%s", "{bucket:.+}", s3a.option.DomainName)).Subrouter())
}
routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())