Merge pull request #36 from chrislusf/master

sync

README.md

@@ -58,11 +58,11 @@ Your support will be really appreciated by me and other supporters!
 Table of Contents
 =================
 
+* [Quick Start](#quick-start)
 * [Introduction](#introduction)
 * [Features](#features)
     * [Additional Features](#additional-features)
     * [Filer Features](#filer-features)
-* [Quick Start](#quick-start)
 * [Example: Using Seaweed Object Store](#example-Using-Seaweed-Object-Store)
 * [Architecture](#architecture)
 * [Compared to Other File Systems](#compared-to-other-file-systems)
@@ -76,6 +76,13 @@ Table of Contents
 * [Benchmark](#Benchmark)
 * [License](#license)
 
+
+## Quick Start ##
+* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
+* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
+
+Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally or a different machine. That is it!
+
 ## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@@ -142,12 +149,6 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 
 [Back to TOC](#table-of-contents)
 
-## Quick Start ##
-* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
-* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
-
-Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally or a different machine. That is it!
-
 ## Example: Using Seaweed Object Store ##
 
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.

docker/Makefile

@@ -12,6 +12,9 @@ build:
 dev: build
 	docker-compose -f local-dev-compose.yml -p seaweedfs up
 
+dev_registry: build
+	docker-compose -f local-registry-compose.yml -p seaweedfs up
+
 cluster: build
 	docker-compose -f local-cluster-compose.yml -p seaweedfs up
 

docker/local-registry-compose.yml

@@ -0,0 +1,95 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    volumes:
+      - type: bind
+        source: /Volumes/mobile_disk/data
+        target: /data
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v 9 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer
+  minio:
+    image: minio/minio
+    ports:
+      - 9000:9000
+    command: 'minio server /data'
+    environment:
+      MINIO_ACCESS_KEY: "some_access_key1"
+      MINIO_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - master
+  registry1:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5001:5001
+    depends_on:
+      - s3
+      - minio
+  registry2:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5002" # minio
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://minio:9000"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5002:5002
+    depends_on:
+      - s3
+      - minio

k8s/seaweedfs/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 2.09
+version: 2.10

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "2.09"
+  imageTag: "2.10"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/src/main/java/seaweedfs/client/ChunkCache.java

@@ -15,7 +15,6 @@ public class ChunkCache {
         }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
-                .weakValues()
                 .expireAfterAccess(1, TimeUnit.HOURS)
                 .build();
     }

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -19,6 +19,7 @@ public class SeaweedRead {
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
 
     static ChunkCache chunkCache = new ChunkCache(4);
+    static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024);
 
     // returns bytesRead
     public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@@ -30,13 +31,19 @@ public class SeaweedRead {
         FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
         for (ChunkView chunkView : chunkViews) {
             String vid = parseVolumeId(chunkView.fileId);
+            if (volumeIdCache.getLocations(vid)==null){
                 lookupRequest.addVolumeIds(vid);
             }
+        }
 
+        if (lookupRequest.getVolumeIdsCount()>0){
             FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
                     .getBlockingStub().lookupVolume(lookupRequest.build());
-
             Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
+            for (Map.Entry<String,FilerProto.Locations> entry : vid2Locations.entrySet()) {
+                volumeIdCache.setLocations(entry.getKey(), entry.getValue());
+            }
+        }
 
         //TODO parallel this
         long readCount = 0;
@@ -50,7 +57,7 @@ public class SeaweedRead {
                 startOffset += gap;
             }
 
-            FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
+            FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId));
             if (locations == null || locations.getLocationsCount() == 0) {
                 LOG.error("failed to locate {}", chunkView.fileId);
                 // log here!

other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@@ -0,0 +1,36 @@
+package seaweedfs.client;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+import java.util.concurrent.TimeUnit;
+
+public class VolumeIdCache {
+
+    private Cache<String, FilerProto.Locations> cache = null;
+
+    public VolumeIdCache(int maxEntries) {
+        if (maxEntries == 0) {
+            return;
+        }
+        this.cache = CacheBuilder.newBuilder()
+                .maximumSize(maxEntries)
+                .expireAfterAccess(1, TimeUnit.HOURS)
+                .build();
+    }
+
+    public FilerProto.Locations getLocations(String volumeId) {
+        if (this.cache == null) {
+            return null;
+        }
+        return this.cache.getIfPresent(volumeId);
+    }
+
+    public void setLocations(String volumeId, FilerProto.Locations locations) {
+        if (this.cache == null) {
+            return;
+        }
+        this.cache.put(volumeId, locations);
+    }
+
+}

other/java/client/src/main/proto/filer.proto

@@ -348,3 +348,20 @@ message KvPutRequest {
 message KvPutResponse {
     string error = 1;
 }
+
+// path-based configurations
+message FilerConf {
+    int32 version = 1;
+    message PathConf {
+        string location_prefix = 1;
+        string collection = 2;
+        string replication = 3;
+        string ttl = 4;
+        enum DiskType {
+            HDD = 0;
+            SSD = 1;
+        }
+        DiskType disk_type = 5;
+    }
+    repeated PathConf locations = 2;
+}

other/java/unzip/pom.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>unzip</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-client</artifactId>
+            <version>1.5.3</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-hadoop2-client</artifactId>
+            <version>1.5.3</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>2.9.2</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+
+</project>

other/java/unzip/src/main/java/com/example/test/Example.java

@@ -0,0 +1,46 @@
+package com.example.test;
+
+import seaweed.hdfs.SeaweedInputStream;
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerGrpcClient;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public class Example {
+
+    public static FilerClient filerClient = new FilerClient("localhost", 18888);
+    public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888);
+
+    public static void main(String[] args) throws IOException {
+
+        // 本地模式，速度很快
+        parseZip("/Users/chris/tmp/test.zip");
+
+        // swfs读取，慢
+        SeaweedInputStream seaweedInputStream = new SeaweedInputStream(
+                filerGrpcClient,
+                new org.apache.hadoop.fs.FileSystem.Statistics(""),
+                "/",
+                filerClient.lookupEntry("/", "test.zip")
+        );
+        parseZip(seaweedInputStream);
+
+    }
+
+    public static void parseZip(String filename) throws IOException {
+        FileInputStream fileInputStream = new FileInputStream(filename);
+        parseZip(fileInputStream);
+    }
+
+    public static void parseZip(InputStream is) throws IOException {
+        ZipInputStream zin = new ZipInputStream(is);
+        ZipEntry ze;
+        while ((ze = zin.getNextEntry()) != null) {
+            System.out.println(ze.getName());
+        }
+    }
+}

weed/command/benchmark.go

@@ -125,7 +125,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", 0, strings.Split(*b.masters, ","))
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", 0, "", strings.Split(*b.masters, ","))
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()
 

weed/command/filer.go

@@ -59,7 +59,7 @@ func init() {
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
-	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
+	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")

weed/command/filer_copy.go

@@ -122,7 +122,7 @@ func runCopy(cmd *Command, args []string) bool {
 			expectedBucket := restPath[:strings.Index(restPath, "/")]
 			if *copy.collection == "" {
 				*copy.collection = expectedBucket
-			} else {
+			} else if *copy.collection != expectedBucket {
 				fmt.Printf("destination %s uses collection \"%s\": unexpected collection \"%v\"\n", urlPath, expectedBucket, *copy.collection)
 				return true
 			}

weed/filer/filer.go

@@ -44,9 +44,9 @@ type Filer struct {
 }
 
 func NewFiler(masters []string, grpcDialOption grpc.DialOption,
-	filerHost string, filerGrpcPort uint32, collection string, replication string, notifyFn func()) *Filer {
+	filerHost string, filerGrpcPort uint32, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
 	f := &Filer{
-		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, masters),
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, dataCenter, masters),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 	}

weed/filer/leveldb/leveldb_store_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}

weed/filer/leveldb2/leveldb2_store_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
@@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}

weed/operation/upload_content.go

@@ -169,7 +169,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
 			_, err = w.Write(data)
 			return
-		}, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
+		}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt)
 	}
 
 	if uploadResult == nil {
@@ -190,6 +190,7 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	body_writer := multipart.NewWriter(buf)
 	h := make(textproto.MIMEHeader)
 	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
+	h.Set("Idempotency-Key", uploadUrl)
 	if mtype == "" {
 		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
 	}

weed/pb/filer.proto

@@ -348,3 +348,20 @@ message KvPutRequest {
 message KvPutResponse {
     string error = 1;
 }
+
+// path-based configurations
+message FilerConf {
+    int32 version = 1;
+    message PathConf {
+        string location_prefix = 1;
+        string collection = 2;
+        string replication = 3;
+        string ttl = 4;
+        enum DiskType {
+            HDD = 0;
+            SSD = 1;
+        }
+        DiskType disk_type = 5;
+    }
+    repeated PathConf locations = 2;
+}

weed/pb/filer_pb/filer.pb.go

@@ -29,6 +29,52 @@ const (
 // of the legacy proto package is being used.
 const _ = proto.ProtoPackageIsVersion4
 
+type FilerConf_PathConf_DiskType int32
+
+const (
+	FilerConf_PathConf_HDD FilerConf_PathConf_DiskType = 0
+	FilerConf_PathConf_SSD FilerConf_PathConf_DiskType = 1
+)
+
+// Enum value maps for FilerConf_PathConf_DiskType.
+var (
+	FilerConf_PathConf_DiskType_name = map[int32]string{
+		0: "HDD",
+		1: "SSD",
+	}
+	FilerConf_PathConf_DiskType_value = map[string]int32{
+		"HDD": 0,
+		"SSD": 1,
+	}
+)
+
+func (x FilerConf_PathConf_DiskType) Enum() *FilerConf_PathConf_DiskType {
+	p := new(FilerConf_PathConf_DiskType)
+	*p = x
+	return p
+}
+
+func (x FilerConf_PathConf_DiskType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (FilerConf_PathConf_DiskType) Descriptor() protoreflect.EnumDescriptor {
+	return file_filer_proto_enumTypes[0].Descriptor()
+}
+
+func (FilerConf_PathConf_DiskType) Type() protoreflect.EnumType {
+	return &file_filer_proto_enumTypes[0]
+}
+
+func (x FilerConf_PathConf_DiskType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use FilerConf_PathConf_DiskType.Descriptor instead.
+func (FilerConf_PathConf_DiskType) EnumDescriptor() ([]byte, []int) {
+	return file_filer_proto_rawDescGZIP(), []int{47, 0, 0}
+}
+
 type LookupDirectoryEntryRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3002,6 +3048,62 @@ func (x *KvPutResponse) GetError() string {
 	return ""
 }
 
+// path-based configurations
+type FilerConf struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Version   int32                 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"`
+	Locations []*FilerConf_PathConf `protobuf:"bytes,2,rep,name=locations,proto3" json:"locations,omitempty"`
+}
+
+func (x *FilerConf) Reset() {
+	*x = FilerConf{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_filer_proto_msgTypes[47]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FilerConf) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FilerConf) ProtoMessage() {}
+
+func (x *FilerConf) ProtoReflect() protoreflect.Message {
+	mi := &file_filer_proto_msgTypes[47]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FilerConf.ProtoReflect.Descriptor instead.
+func (*FilerConf) Descriptor() ([]byte, []int) {
+	return file_filer_proto_rawDescGZIP(), []int{47}
+}
+
+func (x *FilerConf) GetVersion() int32 {
+	if x != nil {
+		return x.Version
+	}
+	return 0
+}
+
+func (x *FilerConf) GetLocations() []*FilerConf_PathConf {
+	if x != nil {
+		return x.Locations
+	}
+	return nil
+}
+
 // if found, send the exact address
 // if not found, send the full list of existing brokers
 type LocateBrokerResponse_Resource struct {
@@ -3016,7 +3118,7 @@ type LocateBrokerResponse_Resource struct {
 func (x *LocateBrokerResponse_Resource) Reset() {
 	*x = LocateBrokerResponse_Resource{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_filer_proto_msgTypes[49]
+		mi := &file_filer_proto_msgTypes[50]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -3029,7 +3131,7 @@ func (x *LocateBrokerResponse_Resource) String() string {
 func (*LocateBrokerResponse_Resource) ProtoMessage() {}
 
 func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[49]
+	mi := &file_filer_proto_msgTypes[50]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -3059,6 +3161,85 @@ func (x *LocateBrokerResponse_Resource) GetResourceCount() int32 {
 	return 0
 }
 
+type FilerConf_PathConf struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	LocationPrefix string                      `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"`
+	Collection     string                      `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
+	Replication    string                      `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"`
+	Ttl            string                      `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"`
+	DiskType       FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"`
+}
+
+func (x *FilerConf_PathConf) Reset() {
+	*x = FilerConf_PathConf{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_filer_proto_msgTypes[51]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FilerConf_PathConf) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FilerConf_PathConf) ProtoMessage() {}
+
+func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message {
+	mi := &file_filer_proto_msgTypes[51]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FilerConf_PathConf.ProtoReflect.Descriptor instead.
+func (*FilerConf_PathConf) Descriptor() ([]byte, []int) {
+	return file_filer_proto_rawDescGZIP(), []int{47, 0}
+}
+
+func (x *FilerConf_PathConf) GetLocationPrefix() string {
+	if x != nil {
+		return x.LocationPrefix
+	}
+	return ""
+}
+
+func (x *FilerConf_PathConf) GetCollection() string {
+	if x != nil {
+		return x.Collection
+	}
+	return ""
+}
+
+func (x *FilerConf_PathConf) GetReplication() string {
+	if x != nil {
+		return x.Replication
+	}
+	return ""
+}
+
+func (x *FilerConf_PathConf) GetTtl() string {
+	if x != nil {
+		return x.Ttl
+	}
+	return ""
+}
+
+func (x *FilerConf_PathConf) GetDiskType() FilerConf_PathConf_DiskType {
+	if x != nil {
+		return x.DiskType
+	}
+	return FilerConf_PathConf_HDD
+}
+
 var File_filer_proto protoreflect.FileDescriptor
 
 var file_filer_proto_rawDesc = []byte{
@@ -3433,7 +3614,28 @@ var file_filer_proto_rawDesc = []byte{
 	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c,
 	0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
 	0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65,
+	0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xcd, 0x02, 0x0a, 0x09, 0x46, 0x69,
+	0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
+	0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
+	0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f,
+	0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xe9, 0x01,
+	0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65,
+	0x66, 0x69, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x42, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f,
+	0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x66, 0x69, 0x6c,
+	0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e,
+	0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70,
+	0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x1c, 0x0a, 0x08, 0x44,
+	0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x00,
+	0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x01, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65,
 	0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f,
 	0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74,
 	0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f,
@@ -3556,123 +3758,129 @@ func file_filer_proto_rawDescGZIP() []byte {
 	return file_filer_proto_rawDescData
 }
 
-var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 50)
+var file_filer_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
+var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 52)
 var file_filer_proto_goTypes = []interface{}{
-	(*LookupDirectoryEntryRequest)(nil),   // 0: filer_pb.LookupDirectoryEntryRequest
+	(FilerConf_PathConf_DiskType)(0),      // 0: filer_pb.FilerConf.PathConf.DiskType
-	(*LookupDirectoryEntryResponse)(nil),  // 1: filer_pb.LookupDirectoryEntryResponse
+	(*LookupDirectoryEntryRequest)(nil),   // 1: filer_pb.LookupDirectoryEntryRequest
-	(*ListEntriesRequest)(nil),            // 2: filer_pb.ListEntriesRequest
+	(*LookupDirectoryEntryResponse)(nil),  // 2: filer_pb.LookupDirectoryEntryResponse
-	(*ListEntriesResponse)(nil),           // 3: filer_pb.ListEntriesResponse
+	(*ListEntriesRequest)(nil),            // 3: filer_pb.ListEntriesRequest
-	(*Entry)(nil),                         // 4: filer_pb.Entry
+	(*ListEntriesResponse)(nil),           // 4: filer_pb.ListEntriesResponse
-	(*FullEntry)(nil),                     // 5: filer_pb.FullEntry
+	(*Entry)(nil),                         // 5: filer_pb.Entry
-	(*EventNotification)(nil),             // 6: filer_pb.EventNotification
+	(*FullEntry)(nil),                     // 6: filer_pb.FullEntry
-	(*FileChunk)(nil),                     // 7: filer_pb.FileChunk
+	(*EventNotification)(nil),             // 7: filer_pb.EventNotification
-	(*FileChunkManifest)(nil),             // 8: filer_pb.FileChunkManifest
+	(*FileChunk)(nil),                     // 8: filer_pb.FileChunk
-	(*FileId)(nil),                        // 9: filer_pb.FileId
+	(*FileChunkManifest)(nil),             // 9: filer_pb.FileChunkManifest
-	(*FuseAttributes)(nil),                // 10: filer_pb.FuseAttributes
+	(*FileId)(nil),                        // 10: filer_pb.FileId
-	(*CreateEntryRequest)(nil),            // 11: filer_pb.CreateEntryRequest
+	(*FuseAttributes)(nil),                // 11: filer_pb.FuseAttributes
-	(*CreateEntryResponse)(nil),           // 12: filer_pb.CreateEntryResponse
+	(*CreateEntryRequest)(nil),            // 12: filer_pb.CreateEntryRequest
-	(*UpdateEntryRequest)(nil),            // 13: filer_pb.UpdateEntryRequest
+	(*CreateEntryResponse)(nil),           // 13: filer_pb.CreateEntryResponse
-	(*UpdateEntryResponse)(nil),           // 14: filer_pb.UpdateEntryResponse
+	(*UpdateEntryRequest)(nil),            // 14: filer_pb.UpdateEntryRequest
-	(*AppendToEntryRequest)(nil),          // 15: filer_pb.AppendToEntryRequest
+	(*UpdateEntryResponse)(nil),           // 15: filer_pb.UpdateEntryResponse
-	(*AppendToEntryResponse)(nil),         // 16: filer_pb.AppendToEntryResponse
+	(*AppendToEntryRequest)(nil),          // 16: filer_pb.AppendToEntryRequest
-	(*DeleteEntryRequest)(nil),            // 17: filer_pb.DeleteEntryRequest
+	(*AppendToEntryResponse)(nil),         // 17: filer_pb.AppendToEntryResponse
-	(*DeleteEntryResponse)(nil),           // 18: filer_pb.DeleteEntryResponse
+	(*DeleteEntryRequest)(nil),            // 18: filer_pb.DeleteEntryRequest
-	(*AtomicRenameEntryRequest)(nil),      // 19: filer_pb.AtomicRenameEntryRequest
+	(*DeleteEntryResponse)(nil),           // 19: filer_pb.DeleteEntryResponse
-	(*AtomicRenameEntryResponse)(nil),     // 20: filer_pb.AtomicRenameEntryResponse
+	(*AtomicRenameEntryRequest)(nil),      // 20: filer_pb.AtomicRenameEntryRequest
-	(*AssignVolumeRequest)(nil),           // 21: filer_pb.AssignVolumeRequest
+	(*AtomicRenameEntryResponse)(nil),     // 21: filer_pb.AtomicRenameEntryResponse
-	(*AssignVolumeResponse)(nil),          // 22: filer_pb.AssignVolumeResponse
+	(*AssignVolumeRequest)(nil),           // 22: filer_pb.AssignVolumeRequest
-	(*LookupVolumeRequest)(nil),           // 23: filer_pb.LookupVolumeRequest
+	(*AssignVolumeResponse)(nil),          // 23: filer_pb.AssignVolumeResponse
-	(*Locations)(nil),                     // 24: filer_pb.Locations
+	(*LookupVolumeRequest)(nil),           // 24: filer_pb.LookupVolumeRequest
-	(*Location)(nil),                      // 25: filer_pb.Location
+	(*Locations)(nil),                     // 25: filer_pb.Locations
-	(*LookupVolumeResponse)(nil),          // 26: filer_pb.LookupVolumeResponse
+	(*Location)(nil),                      // 26: filer_pb.Location
-	(*Collection)(nil),                    // 27: filer_pb.Collection
+	(*LookupVolumeResponse)(nil),          // 27: filer_pb.LookupVolumeResponse
-	(*CollectionListRequest)(nil),         // 28: filer_pb.CollectionListRequest
+	(*Collection)(nil),                    // 28: filer_pb.Collection
-	(*CollectionListResponse)(nil),        // 29: filer_pb.CollectionListResponse
+	(*CollectionListRequest)(nil),         // 29: filer_pb.CollectionListRequest
-	(*DeleteCollectionRequest)(nil),       // 30: filer_pb.DeleteCollectionRequest
+	(*CollectionListResponse)(nil),        // 30: filer_pb.CollectionListResponse
-	(*DeleteCollectionResponse)(nil),      // 31: filer_pb.DeleteCollectionResponse
+	(*DeleteCollectionRequest)(nil),       // 31: filer_pb.DeleteCollectionRequest
-	(*StatisticsRequest)(nil),             // 32: filer_pb.StatisticsRequest
+	(*DeleteCollectionResponse)(nil),      // 32: filer_pb.DeleteCollectionResponse
-	(*StatisticsResponse)(nil),            // 33: filer_pb.StatisticsResponse
+	(*StatisticsRequest)(nil),             // 33: filer_pb.StatisticsRequest
-	(*GetFilerConfigurationRequest)(nil),  // 34: filer_pb.GetFilerConfigurationRequest
+	(*StatisticsResponse)(nil),            // 34: filer_pb.StatisticsResponse
-	(*GetFilerConfigurationResponse)(nil), // 35: filer_pb.GetFilerConfigurationResponse
+	(*GetFilerConfigurationRequest)(nil),  // 35: filer_pb.GetFilerConfigurationRequest
-	(*SubscribeMetadataRequest)(nil),      // 36: filer_pb.SubscribeMetadataRequest
+	(*GetFilerConfigurationResponse)(nil), // 36: filer_pb.GetFilerConfigurationResponse
-	(*SubscribeMetadataResponse)(nil),     // 37: filer_pb.SubscribeMetadataResponse
+	(*SubscribeMetadataRequest)(nil),      // 37: filer_pb.SubscribeMetadataRequest
-	(*LogEntry)(nil),                      // 38: filer_pb.LogEntry
+	(*SubscribeMetadataResponse)(nil),     // 38: filer_pb.SubscribeMetadataResponse
-	(*KeepConnectedRequest)(nil),          // 39: filer_pb.KeepConnectedRequest
+	(*LogEntry)(nil),                      // 39: filer_pb.LogEntry
-	(*KeepConnectedResponse)(nil),         // 40: filer_pb.KeepConnectedResponse
+	(*KeepConnectedRequest)(nil),          // 40: filer_pb.KeepConnectedRequest
-	(*LocateBrokerRequest)(nil),           // 41: filer_pb.LocateBrokerRequest
+	(*KeepConnectedResponse)(nil),         // 41: filer_pb.KeepConnectedResponse
-	(*LocateBrokerResponse)(nil),          // 42: filer_pb.LocateBrokerResponse
+	(*LocateBrokerRequest)(nil),           // 42: filer_pb.LocateBrokerRequest
-	(*KvGetRequest)(nil),                  // 43: filer_pb.KvGetRequest
+	(*LocateBrokerResponse)(nil),          // 43: filer_pb.LocateBrokerResponse
-	(*KvGetResponse)(nil),                 // 44: filer_pb.KvGetResponse
+	(*KvGetRequest)(nil),                  // 44: filer_pb.KvGetRequest
-	(*KvPutRequest)(nil),                  // 45: filer_pb.KvPutRequest
+	(*KvGetResponse)(nil),                 // 45: filer_pb.KvGetResponse
-	(*KvPutResponse)(nil),                 // 46: filer_pb.KvPutResponse
+	(*KvPutRequest)(nil),                  // 46: filer_pb.KvPutRequest
-	nil,                                   // 47: filer_pb.Entry.ExtendedEntry
+	(*KvPutResponse)(nil),                 // 47: filer_pb.KvPutResponse
-	nil,                                   // 48: filer_pb.LookupVolumeResponse.LocationsMapEntry
+	(*FilerConf)(nil),                     // 48: filer_pb.FilerConf
-	(*LocateBrokerResponse_Resource)(nil), // 49: filer_pb.LocateBrokerResponse.Resource
+	nil,                                   // 49: filer_pb.Entry.ExtendedEntry
+	nil,                                   // 50: filer_pb.LookupVolumeResponse.LocationsMapEntry
+	(*LocateBrokerResponse_Resource)(nil), // 51: filer_pb.LocateBrokerResponse.Resource
+	(*FilerConf_PathConf)(nil),            // 52: filer_pb.FilerConf.PathConf
 }
 var file_filer_proto_depIdxs = []int32{
-	4,  // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry
+	5,  // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry
-	4,  // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry
+	5,  // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry
-	7,  // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk
+	8,  // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk
-	10, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes
+	11, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes
-	47, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
+	49, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
-	4,  // 5: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry
+	5,  // 5: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry
-	4,  // 6: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry
+	5,  // 6: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry
-	4,  // 7: filer_pb.EventNotification.new_entry:type_name -> filer_pb.Entry
+	5,  // 7: filer_pb.EventNotification.new_entry:type_name -> filer_pb.Entry
-	9,  // 8: filer_pb.FileChunk.fid:type_name -> filer_pb.FileId
+	10, // 8: filer_pb.FileChunk.fid:type_name -> filer_pb.FileId
-	9,  // 9: filer_pb.FileChunk.source_fid:type_name -> filer_pb.FileId
+	10, // 9: filer_pb.FileChunk.source_fid:type_name -> filer_pb.FileId
-	7,  // 10: filer_pb.FileChunkManifest.chunks:type_name -> filer_pb.FileChunk
+	8,  // 10: filer_pb.FileChunkManifest.chunks:type_name -> filer_pb.FileChunk
-	4,  // 11: filer_pb.CreateEntryRequest.entry:type_name -> filer_pb.Entry
+	5,  // 11: filer_pb.CreateEntryRequest.entry:type_name -> filer_pb.Entry
-	4,  // 12: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry
+	5,  // 12: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry
-	7,  // 13: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk
+	8,  // 13: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk
-	25, // 14: filer_pb.Locations.locations:type_name -> filer_pb.Location
+	26, // 14: filer_pb.Locations.locations:type_name -> filer_pb.Location
-	48, // 15: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
+	50, // 15: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
-	27, // 16: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection
+	28, // 16: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection
-	6,  // 17: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
+	7,  // 17: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
-	49, // 18: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
+	51, // 18: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
-	24, // 19: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations
+	52, // 19: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
-	0,  // 20: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest
+	25, // 20: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations
-	2,  // 21: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest
+	0,  // 21: filer_pb.FilerConf.PathConf.disk_type:type_name -> filer_pb.FilerConf.PathConf.DiskType
-	11, // 22: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest
+	1,  // 22: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest
-	13, // 23: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest
+	3,  // 23: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest
-	15, // 24: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest
+	12, // 24: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest
-	17, // 25: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
+	14, // 25: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest
-	19, // 26: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
+	16, // 26: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest
-	21, // 27: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
+	18, // 27: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
-	23, // 28: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
+	20, // 28: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
-	28, // 29: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
+	22, // 29: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
-	30, // 30: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest
+	24, // 30: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
-	32, // 31: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest
+	29, // 31: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
-	34, // 32: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest
+	31, // 32: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest
-	36, // 33: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest
+	33, // 33: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest
-	36, // 34: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest
+	35, // 34: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest
-	39, // 35: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest
+	37, // 35: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest
-	41, // 36: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest
+	37, // 36: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest
-	43, // 37: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest
+	40, // 37: filer_pb.SeaweedFiler.KeepConnected:input_type -> filer_pb.KeepConnectedRequest
-	45, // 38: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest
+	42, // 38: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest
-	1,  // 39: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
+	44, // 39: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest
-	3,  // 40: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
+	46, // 40: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest
-	12, // 41: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
+	2,  // 41: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
-	14, // 42: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
+	4,  // 42: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
-	16, // 43: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
+	13, // 43: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
-	18, // 44: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
+	15, // 44: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
-	20, // 45: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
+	17, // 45: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
-	22, // 46: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
+	19, // 46: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
-	26, // 47: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
+	21, // 47: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
-	29, // 48: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
+	23, // 48: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
-	31, // 49: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
+	27, // 49: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
-	33, // 50: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
+	30, // 50: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
-	35, // 51: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
+	32, // 51: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
-	37, // 52: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	34, // 52: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
-	37, // 53: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	36, // 53: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
-	40, // 54: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse
+	38, // 54: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	42, // 55: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse
+	38, // 55: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	44, // 56: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
+	41, // 56: filer_pb.SeaweedFiler.KeepConnected:output_type -> filer_pb.KeepConnectedResponse
-	46, // 57: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
+	43, // 57: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse
-	39, // [39:58] is the sub-list for method output_type
+	45, // 58: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
-	20, // [20:39] is the sub-list for method input_type
+	47, // 59: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
-	20, // [20:20] is the sub-list for extension type_name
+	41, // [41:60] is the sub-list for method output_type
-	20, // [20:20] is the sub-list for extension extendee
+	22, // [22:41] is the sub-list for method input_type
-	0,  // [0:20] is the sub-list for field type_name
+	22, // [22:22] is the sub-list for extension type_name
+	22, // [22:22] is the sub-list for extension extendee
+	0,  // [0:22] is the sub-list for field type_name
 }
 
 func init() { file_filer_proto_init() }
@@ -4245,7 +4453,19 @@ func file_filer_proto_init() {
 				return nil
 			}
 		}
-		file_filer_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} {
+		file_filer_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FilerConf); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_filer_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*LocateBrokerResponse_Resource); i {
 			case 0:
 				return &v.state
@@ -4257,19 +4477,32 @@ func file_filer_proto_init() {
 				return nil
 			}
 		}
+		file_filer_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FilerConf_PathConf); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
 	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_filer_proto_rawDesc,
-			NumEnums:      0,
+			NumEnums:      1,
-			NumMessages:   50,
+			NumMessages:   52,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
 		GoTypes:           file_filer_proto_goTypes,
 		DependencyIndexes: file_filer_proto_depIdxs,
+		EnumInfos:         file_filer_proto_enumTypes,
 		MessageInfos:      file_filer_proto_msgTypes,
 	}.Build()
 	File_filer_proto = out.File

weed/pb/master.proto

@@ -130,6 +130,7 @@ message VolumeLocation {
     repeated uint32 new_vids = 3;
     repeated uint32 deleted_vids = 4;
     string leader = 5; // optional when leader is not itself
+    string data_center = 6; // optional when DataCenter is in use
 }
 
 message LookupVolumeRequest {
@@ -187,11 +188,6 @@ message StatisticsResponse {
 //
 // collection related
 //
-
-message StorageType {
-    string replication = 1;
-    string ttl = 2;
-}
 message Collection {
     string name = 1;
 }

weed/s3api/auth_credentials.go

@@ -3,10 +3,11 @@ package s3api
 import (
 	"bytes"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"io/ioutil"
 	"net/http"
 
+	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"github.com/golang/protobuf/jsonpb"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -127,8 +128,14 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 	}
 
 	return func(w http.ResponseWriter, r *http.Request) {
-		errCode := iam.authRequest(r, action)
+		identity, errCode := iam.authRequest(r, action)
 		if errCode == s3err.ErrNone {
+			if identity != nil && identity.Name != "" {
+				r.Header.Set(xhttp.AmzIdentityId, identity.Name)
+				if identity.isAdmin() {
+					r.Header.Set(xhttp.AmzIsAdmin, "true")
+				}
+			}
 			f(w, r)
 			return
 		}
@@ -137,16 +144,16 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 }
 
 // check whether the request has valid access keys
-func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) s3err.ErrorCode {
+func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
 	var identity *Identity
 	var s3Err s3err.ErrorCode
 	var found bool
 	switch getRequestAuthType(r) {
 	case authTypeStreamingSigned:
-		return s3err.ErrNone
+		return identity, s3err.ErrNone
 	case authTypeUnknown:
 		glog.V(3).Infof("unknown auth type")
-		return s3err.ErrAccessDenied
+		return identity, s3err.ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
 		glog.V(3).Infof("v2 auth type")
 		identity, s3Err = iam.isReqAuthenticatedV2(r)
@@ -155,22 +162,22 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		identity, s3Err = iam.reqSignatureV4Verify(r)
 	case authTypePostPolicy:
 		glog.V(3).Infof("post policy auth type")
-		return s3err.ErrNone
+		return identity, s3err.ErrNone
 	case authTypeJWT:
 		glog.V(3).Infof("jwt auth type")
-		return s3err.ErrNotImplemented
+		return identity, s3err.ErrNotImplemented
 	case authTypeAnonymous:
 		identity, found = iam.lookupAnonymous()
 		if !found {
-			return s3err.ErrAccessDenied
+			return identity, s3err.ErrAccessDenied
 		}
 	default:
-		return s3err.ErrNotImplemented
+		return identity, s3err.ErrNotImplemented
 	}
 
 	glog.V(3).Infof("auth error: %v", s3Err)
 	if s3Err != s3err.ErrNone {
-		return s3Err
+		return identity, s3Err
 	}
 
 	glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions)
@@ -178,19 +185,17 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 	bucket, _ := getBucketAndObject(r)
 
 	if !identity.canDo(action, bucket) {
-		return s3err.ErrAccessDenied
+		return identity, s3err.ErrAccessDenied
 	}
 
-	return s3err.ErrNone
+	return identity, s3err.ErrNone
 
 }
 
 func (identity *Identity) canDo(action Action, bucket string) bool {
-	for _, a := range identity.Actions {
+	if identity.isAdmin() {
-		if a == "Admin" {
 		return true
 	}
-	}
 	for _, a := range identity.Actions {
 		if a == action {
 			return true
@@ -207,3 +212,12 @@ func (identity *Identity) canDo(action Action, bucket string) bool {
 	}
 	return false
 }
+
+func (identity *Identity) isAdmin() bool {
+	for _, a := range identity.Actions {
+		if a == "Admin" {
+			return true
+		}
+	}
+	return false
+}

weed/s3api/filer_util.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
@@ -75,6 +76,11 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
 
 }
 
+func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
+	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
+	return filer_pb.GetEntry(s3a, fullPath)
+}
+
 func objectKey(key *string) *string {
 	if strings.HasPrefix(*key, "/") {
 		t := (*key)[1:]

weed/s3api/http/header.go

@@ -28,3 +28,9 @@ const (
 	AmzObjectTagging = "X-Amz-Tagging"
 	AmzTagCount      = "x-amz-tagging-count"
 )
+
+// Non-Standard S3 HTTP request constants
+const (
+	AmzIdentityId = "x-amz-identity-id"
+	AmzIsAdmin = "x-amz-is-admin"  // only set to http request header as a context
+)

weed/s3api/s3api_bucket_handlers.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"encoding/xml"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"math"
 	"net/http"
 	"time"
 
+	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 
@@ -33,9 +35,14 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	identityId := r.Header.Get(xhttp.AmzIdentityId)
+
 	var buckets []*s3.Bucket
 	for _, entry := range entries {
 		if entry.IsDirectory {
+			if !s3a.hasAccess(r, entry) {
+				continue
+			}
 			buckets = append(buckets, &s3.Bucket{
 				Name:         aws.String(entry.Name),
 				CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0).UTC()),
@@ -45,8 +52,8 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 
 	response = ListAllMyBucketsResult{
 		Owner: &s3.Owner{
-			ID:          aws.String(""),
+			ID:          aws.String(identityId),
-			DisplayName: aws.String(""),
+			DisplayName: aws.String(identityId),
 		},
 		Buckets: buckets,
 	}
@@ -80,13 +87,25 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
 		return
 	}
+	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
+		errCode = s3err.ErrBucketAlreadyExists
+	}
 	if errCode != s3err.ErrNone {
 		writeErrorResponse(w, errCode, r.URL)
 		return
 	}
 
+	fn := func(entry *filer_pb.Entry) {
+		if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" {
+			if entry.Extended == nil {
+				entry.Extended = make(map[string][]byte)
+			}
+			entry.Extended[xhttp.AmzIdentityId] = []byte(identityId)
+		}
+	}
+
 	// create the folder for bucket, but lazily create actual collection
-	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, nil); err != nil {
+	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
 		glog.Errorf("PutBucketHandler mkdir: %v", err)
 		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
 		return
@@ -99,7 +118,18 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 
 	bucket, _ := getBucketAndObject(r)
 
-	err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if entry == nil || err == filer_pb.ErrNotFound {
+		writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
+		return
+	}
+
+	if !s3a.hasAccess(r, entry) {
+		writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
+		return
+	}
+
+	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		// delete collection
 		deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
@@ -128,28 +158,34 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 
 	bucket, _ := getBucketAndObject(r)
 
-	err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
-
+	if entry == nil || err == filer_pb.ErrNotFound {
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: s3a.option.BucketsPath,
-			Name:      bucket,
-		}
-
-		glog.V(1).Infof("lookup bucket: %v", request)
-		if _, err := filer_pb.LookupEntry(client, request); err != nil {
-			if err == filer_pb.ErrNotFound {
-				return filer_pb.ErrNotFound
-			}
-			return fmt.Errorf("lookup bucket %s/%s: %v", s3a.option.BucketsPath, bucket, err)
-		}
-
-		return nil
-	})
-
-	if err != nil {
 		writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
 		return
 	}
 
+	if !s3a.hasAccess(r, entry) {
+		writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
+		return
+	}
+
 	writeSuccessResponseEmpty(w)
 }
+
+func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
+	isAdmin := r.Header.Get(xhttp.AmzIsAdmin) != ""
+	if isAdmin {
+		return true
+	}
+	if entry.Extended == nil {
+		return true
+	}
+
+	identityId := r.Header.Get(xhttp.AmzIdentityId)
+	if id, ok := entry.Extended[xhttp.AmzIdentityId]; ok {
+		if identityId != string(id) {
+			return false
+		}
+	}
+	return true
+}

weed/s3api/s3api_object_copy_handlers.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"net/http"
 	"net/url"
@@ -47,6 +48,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 	defer util.CloseResponse(resp)
 
+	glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
 	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
 
 	if errCode != s3err.ErrNone {
@@ -127,6 +129,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	}
 	defer dataReader.Close()
 
+	glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
 	etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)
 
 	if errCode != s3err.ErrNone {

weed/s3api/s3api_object_handlers.go

@@ -323,7 +323,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
 
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {
-		glog.Errorf("upload to filer response read: %v", ra_err)
+		glog.Errorf("upload to filer response read %d: %v", resp.StatusCode, ra_err)
 		return etag, s3err.ErrInternalError
 	}
 	var ret weed_server.FilerPostResult

weed/s3api/s3api_object_multipart_handlers.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"net/http"
 	"net/url"
@@ -28,13 +29,13 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
 		Key:    objectKey(aws.String(object)),
 	})
 
+	glog.V(2).Info("NewMultipartUploadHandler", string(encodeResponse(response)), errCode)
+
 	if errCode != s3err.ErrNone {
 		writeErrorResponse(w, errCode, r.URL)
 		return
 	}
 
-	// println("NewMultipartUploadHandler", string(encodeResponse(response)))
-
 	writeSuccessResponseXML(w, encodeResponse(response))
 
 }
@@ -52,7 +53,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
 		UploadId: aws.String(uploadID),
 	})
 
-	// println("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode)
+	glog.V(2).Info("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode)
 
 	if errCode != s3err.ErrNone {
 		writeErrorResponse(w, errCode, r.URL)
@@ -81,7 +82,7 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	// println("AbortMultipartUploadHandler", string(encodeResponse(response)))
+	glog.V(2).Info("AbortMultipartUploadHandler", string(encodeResponse(response)))
 
 	writeSuccessResponseXML(w, encodeResponse(response))
 
@@ -114,13 +115,14 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 		UploadIdMarker: aws.String(uploadIDMarker),
 	})
 
+	glog.V(2).Info("ListMultipartUploadsHandler", string(encodeResponse(response)), errCode)
+
 	if errCode != s3err.ErrNone {
 		writeErrorResponse(w, errCode, r.URL)
 		return
 	}
 
 	// TODO handle encodingType
-	// println("ListMultipartUploadsHandler", string(encodeResponse(response)))
 
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
@@ -147,13 +149,13 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		UploadId:         aws.String(uploadID),
 	})
 
+	glog.V(2).Info("ListObjectPartsHandler", string(encodeResponse(response)), errCode)
+
 	if errCode != s3err.ErrNone {
 		writeErrorResponse(w, errCode, r.URL)
 		return
 	}
 
-	// println("ListObjectPartsHandler", string(encodeResponse(response)))
-
 	writeSuccessResponseXML(w, encodeResponse(response))
 
 }

weed/server/filer_server.go

@@ -89,7 +89,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		glog.Fatal("master list is required!")
 	}
 
-	fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.Port, option.Collection, option.DefaultReplication, func() {
+	fs.filer = filer.NewFiler(option.Masters, fs.grpcDialOption, option.Host, option.Port, option.Collection, option.DefaultReplication, option.DataCenter, func() {
 		fs.listenersCond.Broadcast()
 	})
 	fs.filer.Cipher = option.Cipher

weed/server/master_grpc_server.go

@@ -88,6 +88,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		message := &master_pb.VolumeLocation{
 			Url:        dn.Url(),
 			PublicUrl:  dn.PublicUrl,
+			DataCenter: string(dn.GetDataCenter().Id()),
 		}
 		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
 			// process delta volume ids if exists for fast volume id updates
@@ -148,7 +149,6 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			}
 
 		}
-
 		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
 			ms.clientChansLock.RLock()
 			for host, ch := range ms.clientChans {

weed/server/master_server.go

@@ -93,7 +93,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *Maste
 		preallocateSize: preallocateSize,
 		clientChans:     make(map[string]chan *master_pb.VolumeLocation),
 		grpcDialOption:  grpcDialOption,
-		MasterClient:    wdclient.NewMasterClient(grpcDialOption, "master", option.Host, 0, peers),
+		MasterClient:    wdclient.NewMasterClient(grpcDialOption, "master", option.Host, 0, "", peers),
 		adminLocks:      NewAdminLocks(),
 	}
 	ms.bounedLeaderChan = make(chan int, 16)

weed/shell/commands.go

@@ -45,7 +45,7 @@ var (
 func NewCommandEnv(options ShellOptions) *CommandEnv {
 	ce := &CommandEnv{
 		env:          make(map[string]string),
-		MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", 0, strings.Split(*options.Masters, ",")),
+		MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", 0, "", strings.Split(*options.Masters, ",")),
 		option:       options,
 	}
 	ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient)

weed/topology/store_replicate.go

@@ -81,6 +81,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId needle.Volume
 			}
 
 			// volume server do not know about encryption
+			// TODO optimize here to compress data only once
 			_, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsCompressed(), string(n.Mime), pairMap, jwt)
 			return err
 		}); err != nil {

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 8)
+	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 10)
 	COMMIT  = ""
 )
 

weed/util/http_util.go

@@ -370,7 +370,6 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e
 	if err != nil {
 		return nil, err
 	}
-	defer CloseResponse(r)
 	if r.StatusCode >= 400 {
 		return nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
 	}

weed/wdclient/masterclient.go

@@ -24,14 +24,14 @@ type MasterClient struct {
 	vidMap
 }
 
-func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost string, clientGrpcPort uint32, masters []string) *MasterClient {
+func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost string, clientGrpcPort uint32, clientDataCenter string, masters []string) *MasterClient {
 	return &MasterClient{
 		clientType:     clientType,
 		clientHost:     clientHost,
 		grpcPort:       clientGrpcPort,
 		masters:        masters,
 		grpcDialOption: grpcDialOption,
-		vidMap:         newVidMap(),
+		vidMap:         newVidMap(clientDataCenter),
 	}
 }
 
@@ -89,7 +89,7 @@ func (mc *MasterClient) tryAllMasters() {
 		}
 
 		mc.currentMaster = ""
-		mc.vidMap = newVidMap()
+		mc.vidMap = newVidMap("")
 	}
 }
 
@@ -132,6 +132,7 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
 			loc := Location{
 				Url:        volumeLocation.Url,
 				PublicUrl:  volumeLocation.PublicUrl,
+				DataCenter: volumeLocation.DataCenter,
 			}
 			for _, newVid := range volumeLocation.NewVids {
 				glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid)

weed/wdclient/vid_map.go

@@ -18,18 +18,20 @@ const (
 type Location struct {
 	Url        string `json:"url,omitempty"`
 	PublicUrl  string `json:"publicUrl,omitempty"`
+	DataCenter string `json:"dataCenter,omitempty"`
 }
 
 type vidMap struct {
 	sync.RWMutex
 	vid2Locations map[uint32][]Location
-
+	DataCenter    string
 	cursor        int32
 }
 
-func newVidMap() vidMap {
+func newVidMap(dataCenter string) vidMap {
 	return vidMap{
 		vid2Locations: make(map[uint32][]Location),
+		DataCenter:    dataCenter,
 		cursor:        -1,
 	}
 }
@@ -56,7 +58,11 @@ func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err er
 		return nil, fmt.Errorf("volume %d not found", id)
 	}
 	for _, loc := range locations {
+		if vc.DataCenter == "" || loc.DataCenter == "" || vc.DataCenter != loc.DataCenter {
 			serverUrls = append(serverUrls, loc.Url)
+		} else {
+			serverUrls = append([]string{loc.Url}, serverUrls...)
+		}
 	}
 	return
 }

weed/wdclient/vid_map_test.go

@@ -45,7 +45,7 @@ func TestLocationIndex(t *testing.T) {
 	mustOk(7, maxCursorIndex, 0)
 
 	// test with constructor
-	vm = newVidMap()
+	vm = newVidMap("")
 	length := 7
 	for i := 0; i < 100; i++ {
 		got, err := vm.getLocationIndex(length)