Merge pull request #75 from chrislusf/master

sync

README.md

@@ -7,6 +7,7 @@
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
 
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
@@ -42,6 +43,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
 - [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
 Table of Contents
@@ -70,7 +72,7 @@ Table of Contents
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
-Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it!
+Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
 
 ## Introduction ##
 

docker/Dockerfile.s3tests

@@ -28,4 +28,4 @@ ENV \
     S3TEST_CONF="/s3test.conf"
 
 ENTRYPOINT ["/bin/bash", "-c"]
-CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]
+CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]

docker/Makefile

@@ -4,8 +4,10 @@ all: gen
 
 gen: dev
 
-build:
+binary:
 	cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static"; mv weed ../docker/
+
+build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed
 
@@ -21,6 +23,9 @@ dev_tls: build certstrap
 dev_mount: build
 	docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
 
+profile_mount: build
+	docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
+
 k8s: build
 	docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
 

docker/compose/local-minio-gateway-compose.yml

@@ -0,0 +1,50 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    depends_on:
+      - master
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8333:8333
+    command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - master
+      - volume
+  minio-gateway-s3:
+    image: minio/minio
+    ports:
+      - 9000:9000
+    command: 'minio gateway s3 http://s3:8333'
+    restart: on-failure
+    environment:
+      MINIO_ACCESS_KEY: "some_access_key1"
+      MINIO_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - s3
+  minio-warp:
+    image: minio/warp
+    command: 'mixed --duration=5m --obj.size=3mb --autoterm'
+    restart: on-failure
+    environment:
+      WARP_HOST: "minio-gateway-s3:9000"
+      WARP_ACCESS_KEY: "some_access_key1"
+      WARP_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - minio-gateway-s3

docker/compose/local-mount-profile-compose.yml

@@ -0,0 +1,47 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 7455:8080
+      - 9325:9325
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 9326:9326
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    command: 'filer -master="master:9333"  -metricsPort=9326'
+    tty: true
+    stdin_open: true
+    depends_on:
+      - master
+      - volume
+  mount:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - fuse
+    volumes:
+      - /Volumes/mobile_disk/99:/data
+    entrypoint: '/bin/sh -c "mkdir -p t1 && weed mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/compose/local-registry-compose.yml

@@ -15,24 +15,18 @@ services:
     command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
     depends_on:
       - master
-  filer:
+  s3:
     image: chrislusf/seaweedfs:local
     ports:
       - 8888:8888
       - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-      - master
-      - volume
-  s3:
-    image: chrislusf/seaweedfs:local
-    ports:
       - 8333:8333
-    command: '-v 9 s3 -filer="filer:8888"'
+    command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
     depends_on:
       - master
       - volume
-      - filer
   minio:
     image: minio/minio
     ports:

docker/compose/local-s3tests-compose.yml

@@ -38,7 +38,7 @@ services:
       S3TEST_CONF: "s3tests.conf"
       NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
       NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
     depends_on:
       - master
       - volume

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/Shopify/sarama v1.23.1
 	github.com/aws/aws-sdk-go v1.34.30
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
+	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
 	github.com/chrislusf/raft v1.0.4
 	github.com/coreos/go-semver v0.3.0 // indirect
@@ -38,7 +39,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.10
@@ -60,7 +61,7 @@ require (
 	github.com/prometheus/client_golang v1.3.0
 	github.com/rakyll/statik v0.1.7
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/seaweedfs/fuse v1.1.1
+	github.com/seaweedfs/fuse v1.1.3
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -88,7 +89,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
 	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
 	google.golang.org/api v0.26.0

go.sum

@@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
@@ -683,6 +685,8 @@ github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw=
 github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
 github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0=
 github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.3 h1:0DddotXwSRGbYG2kynoJyr8GHCy30Z2SpdhP3vdyijY=
+github.com/seaweedfs/fuse v1.1.3/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
 github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
 github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
 github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
@@ -957,6 +961,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

k8s/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.31"
+appVersion: "2.38"
-version: 2.31
+version: 2.38

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.31" - started using {.Chart.appVersion}
+  # imageTag: "2.38" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -17,7 +17,7 @@
         <protobuf.version>3.9.1</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
         <grpc.version>1.23.0</grpc.version>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
     </properties>
 
     <dependencies>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.2</version>
+    <version>1.6.4</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java

@@ -3,6 +3,7 @@ package seaweedfs.client;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,7 +35,7 @@ public class ByteBufferPool {
     }
 
     public static synchronized void release(ByteBuffer obj) {
-        obj.clear();
+        ((Buffer)obj).clear();
         bufferList.add(0, obj);
     }
 

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -94,7 +94,7 @@ public class FilerClient extends FilerGrpcClient {
             return true;
         }
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         mkdirs(parent, mode, uid, gid, userName, groupNames);
@@ -115,11 +115,11 @@ public class FilerClient extends FilerGrpcClient {
     public boolean mv(String oldPath, String newPath) {
 
         File oldPathFile = new File(oldPath);
-        String oldParent = oldPathFile.getParent();
+        String oldParent = oldPathFile.getParent().replace('\\','/');
         String oldName = oldPathFile.getName();
 
         File newPathFile = new File(newPath);
-        String newParent = newPathFile.getParent();
+        String newParent = newPathFile.getParent().replace('\\','/');
         String newName = newPathFile.getName();
 
         return atomicRenameEntry(oldParent, oldName, newParent, newName);
@@ -129,7 +129,7 @@ public class FilerClient extends FilerGrpcClient {
     public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {
 
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         return deleteEntry(
@@ -148,7 +148,7 @@ public class FilerClient extends FilerGrpcClient {
     public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
 
         File pathFile = new File(path);
-        String parent = pathFile.getParent();
+        String parent = pathFile.getParent().replace('\\','/');
         String name = pathFile.getName();
 
         FilerProto.Entry entry = lookupEntry(parent, name);

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.2</version>
+            <version>1.6.4</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.2</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

test/random_access/pom.xml

@@ -8,7 +8,7 @@
     <version>1.0-SNAPSHOT</version>
 
     <properties>
-        <guava.version>28.0-jre</guava.version>
+        <guava.version>30.0-jre</guava.version>
     </properties>
 
     <dependencies>

weed/command/benchmark.go

@@ -305,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		}
 		var bytes []byte
 		for _, url := range urls {
-			bytes, _, err = util.FastGet(url)
+			bytes, _, err = util.Get(url)
 			if err == nil {
 				break
 			}

weed/command/filer.go

@@ -47,6 +47,7 @@ type FilerOptions struct {
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
+	concurrentUploadLimitMB *int
 }
 
 func init() {
@@ -54,12 +55,12 @@ func init() {
 	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
-	f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
-	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
@@ -69,6 +70,7 @@ func init() {
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
+	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -159,21 +161,22 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:            strings.Split(*fo.masters, ","),
+		Masters:               strings.Split(*fo.masters, ","),
-		Collection:         *fo.collection,
+		Collection:            *fo.collection,
-		DefaultReplication: *fo.defaultReplicaPlacement,
+		DefaultReplication:    *fo.defaultReplicaPlacement,
-		DisableDirListing:  *fo.disableDirListing,
+		DisableDirListing:     *fo.disableDirListing,
-		MaxMB:              *fo.maxMB,
+		MaxMB:                 *fo.maxMB,
-		DirListingLimit:    *fo.dirListingLimit,
+		DirListingLimit:       *fo.dirListingLimit,
-		DataCenter:         *fo.dataCenter,
+		DataCenter:            *fo.dataCenter,
-		Rack:               *fo.rack,
+		Rack:                  *fo.rack,
-		DefaultLevelDbDir:  defaultLevelDbDirectory,
+		DefaultLevelDbDir:     defaultLevelDbDirectory,
-		DisableHttp:        *fo.disableHttp,
+		DisableHttp:           *fo.disableHttp,
-		Host:               *fo.ip,
+		Host:                  *fo.ip,
-		Port:               uint32(*fo.port),
+		Port:                  uint32(*fo.port),
-		Cipher:             *fo.cipher,
+		Cipher:                *fo.cipher,
-		SaveToFilerLimit:   *fo.saveToFilerLimit,
+		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
-		Filers:             peers,
+		Filers:                peers,
+		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_cat.go

@@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 
 		filerCat.filerClient = client
 
-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 	})
 

weed/command/filer_copy.go

@@ -56,7 +56,7 @@ func init() {
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
 	copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
 	copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 }

weed/command/master.go

@@ -6,7 +6,6 @@ import (
 	"google.golang.org/grpc/reflection"
 	"net/http"
 	"os"
-	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -48,8 +47,8 @@ type MasterOptions struct {
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address")
+	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
-	m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@@ -86,7 +85,6 @@ func runMaster(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
 	grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
 
 	parent, _ := util.FullPath(*m.metaFolder).DirAndName()

weed/command/mount.go

@@ -25,6 +25,7 @@ type MountOptions struct {
 	volumeServerAccess *string
 	uidMap             *string
 	gidMap             *string
+	readOnly           *bool
 }
 
 var (
@@ -45,7 +46,7 @@ func init() {
 	mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
-	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0")
+	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
@@ -55,6 +56,7 @@ func init() {
 	mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
+	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
 
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

weed/command/mount_std.go

@@ -75,7 +75,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		if err != nil {
 			glog.V(0).Infof("failed to talk to filer %s: %v", filerGrpcAddress, err)
 			glog.V(0).Infof("wait for %d seconds ...", i+1)
-			time.Sleep(time.Duration(i+1)*time.Second)
+			time.Sleep(time.Duration(i+1) * time.Second)
 		}
 	}
 	if err != nil {
@@ -203,6 +203,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		VolumeServerAccess: *mountOptions.volumeServerAccess,
 		Cipher:             cipher,
 		UidGidMapper:       uidGidMapper,
+		ReadOnly:           *option.readOnly,
 	})
 
 	// mount

weed/command/scaffold.go

@@ -103,9 +103,9 @@ dir = "./filerrdb"					# directory to store rocksdb files
 
 [mysql]  # or memsql, tidb
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000)  COMMENT 'directory or file name',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
-#   directory   TEXT           COMMENT 'full path to parent directory',
+#   directory   TEXT                 COMMENT 'full path to parent directory',
 #   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
@@ -120,13 +120,16 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
   CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
     dirhash BIGINT,
-    name VARCHAR(1000),
+    name VARCHAR(1000) BINARY,
     directory TEXT,
     meta LONGBLOB,
     PRIMARY KEY (dirhash, name)
@@ -141,6 +144,9 @@ connection_max_idle = 2
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
 interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
 
 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
@@ -161,6 +167,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [postgres2]
 enabled = false
@@ -183,6 +192,9 @@ sslmode = "disable"
 connection_max_idle = 100
 connection_max_open = 100
 connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
 
 [cassandra]
 # CREATE TABLE filemeta (
@@ -506,7 +518,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs
 
 
 [master.sequencer]
-type = "raft"     # Choose [raft|etcd] type for storing the file id sequence
+type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
 # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
 sequencer_etcd_urls = "http://127.0.0.1:2379"

weed/command/server.go

@@ -2,9 +2,8 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"os"
-	"runtime"
-	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -16,6 +15,7 @@ import (
 
 type ServerOptions struct {
 	cpuprofile *string
+	memprofile *string
 	v          VolumeServerOptions
 }
 
@@ -49,8 +49,8 @@ var cmdServer = &Command{
 }
 
 var (
-	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
+	serverIp                  = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
-	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	serverBindIp              = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
 	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
@@ -76,6 +76,7 @@ var (
 
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
+	serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
 
 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@@ -93,11 +94,12 @@ func init() {
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
-	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
 	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
+	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
@@ -107,6 +109,7 @@ func init() {
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
@@ -138,14 +141,7 @@ func runServer(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
-	if *serverOptions.cpuprofile != "" {
+	grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)
-		f, err := os.Create(*serverOptions.cpuprofile)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-	}
 
 	if *isStartingS3 {
 		*isStartingFiler = true
@@ -192,7 +188,6 @@ func runServer(cmd *Command, args []string) bool {
 	webdavOptions.filer = &filerAddress
 	msgBrokerOptions.filer = &filerAddress
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
 	go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
 
 	folders := strings.Split(*volumeDataFolders, ",")

weed/command/upload.go

@@ -43,7 +43,7 @@ func init() {
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
 	upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
 }
 

weed/command/volume.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
-	"runtime"
 	"runtime/pprof"
 	"strconv"
 	"strings"
@@ -36,31 +35,32 @@ var (
 )
 
 type VolumeServerOptions struct {
-	port                  *int
+	port                    *int
-	publicPort            *int
+	publicPort              *int
-	folders               []string
+	folders                 []string
-	folderMaxLimits       []int
+	folderMaxLimits         []int
-	idxFolder             *string
+	idxFolder               *string
-	ip                    *string
+	ip                      *string
-	publicUrl             *string
+	publicUrl               *string
-	bindIp                *string
+	bindIp                  *string
-	masters               *string
+	masters                 *string
-	idleConnectionTimeout *int
+	idleConnectionTimeout   *int
-	dataCenter            *string
+	dataCenter              *string
-	rack                  *string
+	rack                    *string
-	whiteList             []string
+	whiteList               []string
-	indexType             *string
+	indexType               *string
-	diskType              *string
+	diskType                *string
-	fixJpgOrientation     *bool
+	fixJpgOrientation       *bool
-	readRedirect          *bool
+	readRedirect            *bool
-	cpuProfile            *string
+	cpuProfile              *string
-	memProfile            *string
+	memProfile              *string
-	compactionMBPerSecond *int
+	compactionMBPerSecond   *int
-	fileSizeLimitMB       *int
+	fileSizeLimitMB         *int
-	minFreeSpacePercents  []float32
+	concurrentUploadLimitMB *int
-	pprof                 *bool
+	minFreeSpacePercents    []float32
-	preStopSeconds        *int
+	pprof                   *bool
-	metricsHttpPort       *int
+	preStopSeconds          *int
+	metricsHttpPort         *int
 	// pulseSeconds          *int
 	enableTcp *bool
 }
@@ -69,9 +69,9 @@ func init() {
 	cmdVolume.Run = runVolume // break init cycle
 	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
-	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name")
+	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
-	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
 	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
@@ -86,6 +86,7 @@ func init() {
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
+	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
@@ -111,8 +112,6 @@ func runVolume(cmd *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
 	// If --pprof is set we assume the caller wants to be able to collect
 	// cpu and memory profiles via go tool pprof
 	if !*v.pprof {
@@ -240,6 +239,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		*v.fixJpgOrientation, *v.readRedirect,
 		*v.compactionMBPerSecond,
 		*v.fileSizeLimitMB,
+		int64(*v.concurrentUploadLimitMB)*1024*1024,
 	)
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)

weed/filer/abstract_sql/abstract_sql_store.go

@@ -13,15 +13,15 @@ import (
 )
 
 type SqlGenerator interface {
-	GetSqlInsert(bucket string) string
+	GetSqlInsert(tableName string) string
-	GetSqlUpdate(bucket string) string
+	GetSqlUpdate(tableName string) string
-	GetSqlFind(bucket string) string
+	GetSqlFind(tableName string) string
-	GetSqlDelete(bucket string) string
+	GetSqlDelete(tableName string) string
-	GetSqlDeleteFolderChildren(bucket string) string
+	GetSqlDeleteFolderChildren(tableName string) string
-	GetSqlListExclusive(bucket string) string
+	GetSqlListExclusive(tableName string) string
-	GetSqlListInclusive(bucket string) string
+	GetSqlListInclusive(tableName string) string
-	GetSqlCreateTable(bucket string) string
+	GetSqlCreateTable(tableName string) string
-	GetSqlDropTable(bucket string) string
+	GetSqlDropTable(tableName string) string
 }
 
 type AbstractSqlStore struct {
@@ -32,6 +32,29 @@ type AbstractSqlStore struct {
 	dbsLock            sync.Mutex
 }
 
+func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.CreateTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	store.dbs[bucket] = true
+}
+func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.deleteTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	delete(store.dbs, bucket)
+}
+
 const (
 	DEFAULT_TABLE = "filemeta"
 )

weed/filer/filechunk_manifest.go

@@ -97,20 +97,20 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
 func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
 
 	var err error
-	var buffer bytes.Buffer
 	var shouldRetry bool
+	receivedData := make([]byte, 0, size)
 
 	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
 		for _, urlString := range urlStrings {
-			shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+			receivedData = receivedData[:0]
-				buffer.Write(data)
+			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				receivedData = append(receivedData, data...)
 			})
 			if !shouldRetry {
 				break
 			}
 			if err != nil {
 				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
-				buffer.Reset()
 			} else {
 				break
 			}
@@ -123,7 +123,8 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
 		}
 	}
 
-	return buffer.Bytes(), err
+	return receivedData, err
+
 }
 
 func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {

weed/filer/filer.go

@@ -241,12 +241,12 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
 	if oldEntry != nil {
 		entry.Attr.Crtime = oldEntry.Attr.Crtime
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
-			glog.Errorf("existing %s is a directory", entry.FullPath)
+			glog.Errorf("existing %s is a directory", oldEntry.FullPath)
-			return fmt.Errorf("existing %s is a directory", entry.FullPath)
+			return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
 		}
 		if !oldEntry.IsDirectory() && entry.IsDirectory() {
-			glog.Errorf("existing %s is a file", entry.FullPath)
+			glog.Errorf("existing %s is a file", oldEntry.FullPath)
-			return fmt.Errorf("existing %s is a file", entry.FullPath)
+			return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
 		}
 	}
 	return f.Store.UpdateEntry(ctx, entry)

weed/filer/filer_on_meta_event.go

@@ -12,6 +12,24 @@ import (
 // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
 func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
 	f.maybeReloadFilerConfiguration(event)
+	f.onBucketEvents(event)
+}
+
+func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
+	message := event.EventNotification
+	for _, sig := range message.Signatures {
+		if sig == f.Signature {
+			return
+		}
+	}
+	if f.DirBucketsPath == event.Directory {
+		if message.OldEntry == nil && message.NewEntry != nil {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
+		}
+		if message.OldEntry != nil && message.NewEntry == nil {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+	}
 }
 
 func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
@@ -34,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
 
 func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
 	if err != nil {
 		return nil, err
 	}

weed/filer/filerstore.go

@@ -39,3 +39,8 @@ type FilerStore interface {
 
 	Shutdown()
 }
+
+type BucketAware interface {
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
+}

weed/filer/filerstore_wrapper.go

@@ -21,6 +21,8 @@ type VirtualFilerStore interface {
 	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
 	DeleteOneEntry(ctx context.Context, entry *Entry) error
 	AddPathSpecificStore(path string, storeId string, store FilerStore)
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
 }
 
 type FilerStoreWrapper struct {
@@ -40,6 +42,27 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
 	}
 }
 
+func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketCreation(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketCreation(bucket)
+	}
+}
+func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketDeletion(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketDeletion(bucket)
+	}
+}
+
 func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
 	fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store)
 	err := fsw.pathToStore.Put([]byte(path), storeId)

weed/filer/leveldb/leveldb_store.go

@@ -8,6 +8,7 @@ import (
 	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -38,6 +39,7 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre
 
 func (store *LevelDBStore) initialize(dir string) (err error) {
 	glog.Infof("filer store dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/leveldb2/leveldb2_store.go

@@ -38,6 +38,7 @@ func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, pr
 
 func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 	glog.Infof("filer store leveldb2 dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/leveldb3/leveldb3_store.go

@@ -45,6 +45,7 @@ func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, pr
 
 func (store *LevelDB3Store) initialize(dir string) (err error) {
 	glog.Infof("filer store leveldb3 dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/mysql/mysql_sql_gen.go

@@ -2,6 +2,7 @@ package mysql
 
 import (
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	_ "github.com/go-sql-driver/mysql"
 )
@@ -9,44 +10,49 @@ import (
 type SqlGenMysql struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }
 
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
 )
 
-func (gen *SqlGenMysql) GetSqlInsert(bucket string) string {
+func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
-	return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket)
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
+	}
 }
 
-func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string {
+func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
-	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket)
+	return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlFind(bucket string) string {
+func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
-	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+	return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDelete(bucket string) string {
+func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string {
+func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", bucket)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string {
+func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string {
+func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
-	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket)
+	return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string {
+func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }
 
-func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string {
+func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }

weed/filer/mysql/mysql_store.go

@@ -3,9 +3,10 @@ package mysql
 import (
 	"database/sql"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer"
+
 	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	_ "github.com/go-sql-driver/mysql"
@@ -29,6 +30,8 @@ func (store *MysqlStore) GetName() string {
 
 func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -41,13 +44,17 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
 	)
 }
 
-func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {
 
 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenMysql{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

weed/filer/mysql2/mysql2_store.go

@@ -32,6 +32,8 @@ func (store *MysqlStore2) GetName() string {
 func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -44,13 +46,17 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
 	)
 }
 
-func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen,
+func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
 	maxLifetimeSeconds int, interpolateParams bool) (err error) {
 
 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &mysql.SqlGenMysql{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   "drop table `%s`",
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

weed/filer/postgres/postgres_sql_gen.go

@@ -10,44 +10,49 @@ import (
 type SqlGenPostgres struct {
 	CreateTableSqlTemplate string
 	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
 }
 
 var (
 	_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
 )
 
-func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
-	return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, bucket)
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
+	}
 }
 
-func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
-	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, bucket)
+	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlFind(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
-	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket)
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
-	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, bucket)
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
-	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket)
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
-	return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket)
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
 }
 
-func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string {
+func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
-	return fmt.Sprintf(gen.DropTableSqlTemplate, bucket)
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
 }

weed/filer/postgres/postgres_store.go

@@ -29,6 +29,8 @@ func (store *PostgresStore) GetName() string {
 
 func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -42,12 +44,16 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
 	)
 }
 
-func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
 
 	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &SqlGenPostgres{
 		CreateTableSqlTemplate: "",
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

weed/filer/postgres2/postgres2_store.go

@@ -32,6 +32,8 @@ func (store *PostgresStore2) GetName() string {
 func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
 		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetString(prefix+"hostname"),
@@ -45,12 +47,16 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
 	)
 }
 
-func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
 
 	store.SupportBucketTable = true
+	if !enableUpsert {
+		upsertQuery = ""
+	}
 	store.SqlGenerator = &postgres.SqlGenPostgres{
 		CreateTableSqlTemplate: createTable,
 		DropTableSqlTemplate:   `drop table "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
 	}
 
 	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

weed/filer/read_write.go

@@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}
 
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 }
 
@@ -35,7 +35,7 @@ func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
 
 	target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
 
-	data, _, err := util.FastGet(target)
+	data, _, err := util.Get(target)
 
 	return data, err
 }

weed/filer/rocksdb/rocksdb_store.go

@@ -8,6 +8,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/tecbot/gorocksdb"
 
@@ -56,6 +57,7 @@ func (store *RocksDBStore) Initialize(configuration weed_util.Configuration, pre
 
 func (store *RocksDBStore) initialize(dir string) (err error) {
 	glog.Infof("filer store rocksdb dir: %s", dir)
+	os.MkdirAll(dir, 0755)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}

weed/filer/stream.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"bytes"
 	"fmt"
+	"golang.org/x/sync/errgroup"
 	"io"
 	"math"
 	"strings"
@@ -13,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
-func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
+func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error {
 
 	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@@ -33,10 +34,20 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 		fileId2Url[chunkView.FileId] = urlStrings
 	}
 
+	if isCheck {
+		// Pre-check all chunkViews urls
+		gErr := new(errgroup.Group)
+		CheckAllChunkViews(chunkViews, &fileId2Url, gErr)
+		if err := gErr.Wait(); err != nil {
+			glog.Errorf("check all chunks: %v", err)
+			return fmt.Errorf("check all chunks: %v", err)
+		}
+		return nil
+	}
+
 	for _, chunkView := range chunkViews {
 
 		urlStrings := fileId2Url[chunkView.FileId]
-
 		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
 		if err != nil {
 			glog.Errorf("read chunk: %v", err)
@@ -54,6 +65,17 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 
 }
 
+func CheckAllChunkViews(chunkViews []*ChunkView, fileId2Url *map[string][]string, gErr *errgroup.Group) {
+	for _, chunkView := range chunkViews {
+		urlStrings := (*fileId2Url)[chunkView.FileId]
+		glog.V(9).Infof("Check chunk: %+v\n url: %v", chunkView, urlStrings)
+		gErr.Go(func() error {
+			_, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+			return err
+		})
+	}
+}
+
 // ----------------  ReadAllReader ----------------------------------
 
 func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) ([]byte, error) {
@@ -185,7 +207,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	var buffer bytes.Buffer
 	var shouldRetry bool
 	for _, urlString := range urlStrings {
-		shouldRetry, err = util.FastReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			buffer.Write(data)
 		})
 		if !shouldRetry {

weed/filesys/dir.go

@@ -128,6 +128,10 @@ func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.N
 func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
 
+	if dir.wfs.option.ReadOnly {
+		return nil, nil, fuse.EPERM
+	}
+
 	request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, req.Flags&fuse.OpenExclusive != 0)
 
 	if err != nil {
@@ -148,6 +152,10 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 
 func (dir *Dir) Mknod(ctx context.Context, req *fuse.MknodRequest) (fs.Node, error) {
 
+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	request, err := dir.doCreateEntry(req.Name, req.Mode, req.Uid, req.Gid, false)
 
 	if err != nil {
@@ -202,6 +210,10 @@ func (dir *Dir) doCreateEntry(name string, mode os.FileMode, uid, gid uint32, ex
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
 
+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	glog.V(4).Infof("mkdir %s: %s", dir.FullPath(), req.Name)
 
 	newEntry := &filer_pb.Entry{
@@ -356,6 +368,11 @@ func findFileType(mode uint16) fuse.DirentType {
 
 func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
+
 	if !req.Dir {
 		return dir.removeOneFile(req)
 	}
@@ -429,6 +446,10 @@ func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
 
 func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("%v dir setattr %+v", dir.FullPath(), req)
 
 	if err := dir.maybeLoadEntry(); err != nil {
@@ -457,6 +478,10 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 
 func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
 
+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("dir Setxattr %s: %s", dir.FullPath(), req.Name)
 
 	if err := dir.maybeLoadEntry(); err != nil {
@@ -473,6 +498,10 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
 
 func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
 
+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("dir Removexattr %s: %s", dir.FullPath(), req.Name)
 
 	if err := dir.maybeLoadEntry(); err != nil {

weed/filesys/dir_link.go

@@ -24,6 +24,10 @@ const (
 
 func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (fs.Node, error) {
 
+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	oldFile, ok := old.(*File)
 	if !ok {
 		glog.Errorf("old node is not a file: %+v", old)
@@ -105,6 +109,10 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f
 
 func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
 
+	if dir.wfs.option.ReadOnly {
+		return nil, fuse.EPERM
+	}
+
 	glog.V(4).Infof("Symlink: %v/%v to %v", dir.FullPath(), req.NewName, req.Target)
 
 	request := &filer_pb.CreateEntryRequest{

weed/filesys/dir_rename.go

@@ -13,6 +13,10 @@ import (
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
 
+	if dir.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	newDir := newDirectory.(*Dir)
 
 	newPath := util.NewFullPath(newDir.FullPath(), req.NewName)

weed/filesys/dir_test.go

@@ -1,34 +0,0 @@
-package filesys
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestDirPath(t *testing.T) {
-
-	p := &Dir{name: "/some"}
-	p = &Dir{name: "path", parent: p}
-	p = &Dir{name: "to", parent: p}
-	p = &Dir{name: "a", parent: p}
-	p = &Dir{name: "file", parent: p}
-
-	assert.Equal(t, "/some/path/to/a/file", p.FullPath())
-
-	p = &Dir{name: "/some"}
-	assert.Equal(t, "/some", p.FullPath())
-
-	p = &Dir{name: "/"}
-	assert.Equal(t, "/", p.FullPath())
-
-	p = &Dir{name: "/"}
-	p = &Dir{name: "path", parent: p}
-	assert.Equal(t, "/path", p.FullPath())
-
-	p = &Dir{name: "/"}
-	p = &Dir{name: "path", parent: p}
-	p = &Dir{name: "to", parent: p}
-	assert.Equal(t, "/path/to", p.FullPath())
-
-}

weed/filesys/file.go

@@ -56,6 +56,10 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {
 		}
 	}
 
+	if entry == nil {
+		return fuse.ENOENT
+	}
+
 	// attr.Inode = file.fullpath().AsInode()
 	attr.Valid = time.Second
 	attr.Mode = os.FileMode(entry.Attributes.FileMode)
@@ -106,6 +110,10 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
 
 func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req)
 
 	entry, err := file.maybeLoadEntry(ctx)
@@ -147,7 +155,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 			}
 			entry.Chunks = chunks
 			file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
-			file.reader = nil
+			file.setReader(nil)
 		}
 		entry.Attributes.FileSize = req.Size
 		file.dirtyMetadata = true
@@ -196,6 +204,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 
 func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
 
+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)
 
 	entry, err := file.maybeLoadEntry(ctx)
@@ -213,6 +225,10 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error
 
 func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
 
+	if file.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)
 
 	entry, err := file.maybeLoadEntry(ctx)
@@ -257,6 +273,8 @@ func (file *File) Forget() {
 	t := util.NewFullPath(file.dir.FullPath(), file.Name)
 	glog.V(4).Infof("Forget file %s", t)
 	file.wfs.fsNodeCache.DeleteFsNode(t)
+	file.wfs.ReleaseHandle(t, 0)
+	file.setReader(nil)
 }
 
 func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
@@ -323,19 +341,29 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 		file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
 	}
 
-	file.reader = nil
+	file.setReader(nil)
 
 	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))
 
 	entry.Chunks = append(entry.Chunks, newChunks...)
 }
 
+func (file *File) setReader(reader io.ReaderAt) {
+	r := file.reader
+	if r != nil {
+		if closer, ok := r.(io.Closer); ok {
+			closer.Close()
+		}
+	}
+	file.reader = reader
+}
+
 func (file *File) setEntry(entry *filer_pb.Entry) {
 	file.entryLock.Lock()
 	defer file.entryLock.Unlock()
 	file.entry = entry
 	file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
-	file.reader = nil
+	file.setReader(nil)
 }
 
 func (file *File) clearEntry() {
@@ -343,7 +371,7 @@ func (file *File) clearEntry() {
 	defer file.entryLock.Unlock()
 	file.entry = nil
 	file.entryViewCache = nil
-	file.reader = nil
+	file.setReader(nil)
 }
 
 func (file *File) saveEntry(entry *filer_pb.Entry) error {

weed/filesys/filehandle.go

@@ -23,7 +23,7 @@ type FileHandle struct {
 	dirtyPages  *ContinuousDirtyPages
 	contentType string
 	handle      uint64
-	sync.RWMutex
+	sync.Mutex
 
 	f         *File
 	RequestId fuse.RequestID // unique ID for request
@@ -59,8 +59,8 @@ var _ = fs.HandleReleaser(&FileHandle{})
 func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
 
 	glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
-	fh.RLock()
+	fh.Lock()
-	defer fh.RUnlock()
+	defer fh.Unlock()
 
 	if req.Size <= 0 {
 		return nil
@@ -130,7 +130,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		if chunkResolveErr != nil {
 			return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
 		}
-		fh.f.reader = nil
+		fh.f.setReader(nil)
 	}
 
 	reader := fh.f.reader
@@ -138,7 +138,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 		chunkViews := filer.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64)
 		reader = filer.NewChunkReaderAtFromClient(fh.f.wfs.LookupFn(), chunkViews, fh.f.wfs.chunkCache, fileSize)
 	}
-	fh.f.reader = reader
+	fh.f.setReader(reader)
 
 	totalRead, err := reader.ReadAt(buff, offset)
 
@@ -154,6 +154,10 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 // Write to the file handle
 func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
 
+	if fh.f.wfs.option.ReadOnly {
+		return fuse.EPERM
+	}
+
 	fh.Lock()
 	defer fh.Unlock()
 
@@ -207,12 +211,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 		fh.f.isOpen--
 
 		fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
-		if closer, ok := fh.f.reader.(io.Closer); ok {
+		fh.f.setReader(nil)
-			if closer != nil {
-				closer.Close()
-			}
-		}
-		fh.f.reader = nil
 	}
 
 	return nil

weed/filesys/wfs.go

@@ -43,6 +43,7 @@ type Option struct {
 	DataCenter         string
 	EntryCacheTtl      time.Duration
 	Umask              os.FileMode
+	ReadOnly           bool
 
 	MountUid   uint32
 	MountGid   uint32

weed/operation/buffer_pool.go

@@ -0,0 +1,24 @@
+package operation
+
+import (
+	"github.com/valyala/bytebufferpool"
+	"sync/atomic"
+)
+
+var bufferCounter int64
+
+func GetBuffer() *bytebufferpool.ByteBuffer {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, 1)
+		// println("+", bufferCounter)
+	}()
+	return bytebufferpool.Get()
+}
+
+func PutBuffer(buf *bytebufferpool.ByteBuffer) {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, -1)
+		// println("-", bufferCounter)
+	}()
+	bytebufferpool.Put(buf)
+}

weed/operation/upload_content.go

@@ -19,7 +19,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/valyala/bytebufferpool"
 )
 
 type UploadResult struct {
@@ -190,8 +189,8 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 }
 
 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
-	buf := bytebufferpool.Get()
+	buf := GetBuffer()
-	defer bytebufferpool.Put(buf)
+	defer PutBuffer(buf)
 	body_writer := multipart.NewWriter(buf)
 	h := make(textproto.MIMEHeader)
 	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))

weed/pb/filer_pb/filer.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: filer.proto
 

weed/pb/filer_pb/filer_client.go

@@ -104,7 +104,7 @@ func SeaweedList(client SeaweedFilerClient, parentDirectoryPath, prefix string,
 func doSeaweedList(client SeaweedFilerClient, fullDirPath util.FullPath, prefix string, fn EachEntryFunciton, startFrom string, inclusive bool, limit uint32) (err error) {
 	// Redundancy limit to make it correctly judge whether it is the last file.
 	redLimit := limit
-	if limit != math.MaxInt32 && limit != 0{
+	if limit != math.MaxInt32 && limit != 0 {
 		redLimit = limit + 1
 	}
 	request := &ListEntriesRequest{
@@ -182,6 +182,26 @@ func Exists(filerClient FilerClient, parentDirectoryPath string, entryName strin
 	return
 }
 
+func Touch(filerClient FilerClient, parentDirectoryPath string, entryName string, entry *Entry) (err error) {
+
+	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
+
+		request := &UpdateEntryRequest{
+			Directory: parentDirectoryPath,
+			Entry:     entry,
+		}
+
+		glog.V(4).Infof("touch entry %v/%v: %v", parentDirectoryPath, entryName, request)
+		if err := UpdateEntry(client, request); err != nil {
+			glog.V(0).Infof("touch exists entry %v: %v", request, err)
+			return fmt.Errorf("touch exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
+		}
+
+		return nil
+	})
+
+}
+
 func Mkdir(filerClient FilerClient, parentDirectoryPath string, dirName string, fn func(entry *Entry)) error {
 	return filerClient.WithFilerClient(func(client SeaweedFilerClient) error {
 

weed/pb/iam_pb/iam.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: iam.proto
 

weed/pb/master_pb/master.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: master.proto
 

weed/pb/messaging_pb/messaging.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.24.0
+// 	protoc-gen-go v1.25.0
 // 	protoc        v3.12.3
 // source: messaging.proto
 

weed/pb/volume_server.proto

@@ -52,6 +52,11 @@ service VolumeServer {
     rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
     }
 
+    rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
+    }
+    rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
+    }
+
     rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
     }
     rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
@@ -253,6 +258,25 @@ message CopyFileResponse {
     bytes file_content = 1;
 }
 
+message ReadNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleBlobResponse {
+    bytes needle_blob = 1;
+}
+
+message WriteNeedleBlobRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int32 size = 3;
+    bytes needle_blob = 4;
+}
+message WriteNeedleBlobResponse {
+}
+
 message VolumeTailSenderRequest {
     uint32 volume_id = 1;
     uint64 since_ns = 2;

weed/replication/repl_util/replication_util.go

@@ -20,7 +20,7 @@ func CopyFromChunkViews(chunkViews []*filer.ChunkView, filerSource *source.Filer
 		var shouldRetry bool
 
 		for _, fileUrl := range fileUrls {
-			shouldRetry, err = util.FastReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
+			shouldRetry, err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
 				writeErr = writeFunc(data)
 			})
 			if err != nil {

weed/s3api/filer_util.go

@@ -79,6 +79,12 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
 
 }
 
+func (s3a *S3ApiServer) touch(parentDirectoryPath string, entryName string, entry *filer_pb.Entry) (err error) {
+
+	return filer_pb.Touch(s3a, parentDirectoryPath, entryName, entry)
+
+}
+
 func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *filer_pb.Entry, err error) {
 	fullPath := util.NewFullPath(parentDirectoryPath, entryName)
 	return filer_pb.GetEntry(s3a, fullPath)

weed/s3api/s3api_object_copy_handlers.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -25,6 +26,26 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
+
+	if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
+		fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
+		dir, name := fullPath.DirAndName()
+		entry, err := s3a.getEntry(dir, name)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
+		err = s3a.touch(dir, name, entry)
+		if err != nil {
+			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		}
+		writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
+			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
+			LastModified: time.Now().UTC(),
+		}))
+		return
+	}
+
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
 		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
@@ -32,7 +53,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	if srcBucket == dstBucket && srcObject == dstObject {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
 		return
 	}
 
@@ -147,3 +168,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseXML(w, encodeResponse(response))
 
 }
+
+func isReplace(r *http.Request) bool {
+	return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
+}

weed/s3api/s3api_object_handlers.go

@@ -311,7 +311,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 	}
 	defer util.CloseResponse(resp)
 
-	if (resp.ContentLength == -1 || resp.StatusCode == 404) && !strings.HasSuffix(destUrl, "/") {
+	if resp.ContentLength == -1 || resp.StatusCode == 404 {
 		if r.Method != "DELETE" {
 			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
 			return
@@ -326,7 +326,11 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
 	for k, v := range proxyResponse.Header {
 		w.Header()[k] = v
 	}
-	w.WriteHeader(proxyResponse.StatusCode)
+	if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
+		w.WriteHeader(http.StatusPartialContent)
+	} else {
+		w.WriteHeader(proxyResponse.StatusCode)
+	}
 	io.Copy(w, proxyResponse.Body)
 }
 

weed/s3api/s3api_objects_list_handlers.go

@@ -206,7 +206,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 		isTruncated = isTruncated || subIsTruncated
 		maxKeys -= subCounter
 		nextMarker = subDir + "/" + subNextMarker
-		counter += subCounter
 		// finished processing this sub directory
 		marker = subDir
 	}

weed/sequence/snowflake_sequencer.go

@@ -0,0 +1,46 @@
+package sequence
+
+import (
+	"fmt"
+	"hash/fnv"
+
+	"github.com/bwmarrin/snowflake"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+// a simple snowflake Sequencer
+type SnowflakeSequencer struct {
+	node *snowflake.Node
+}
+
+func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
+	nodeid_hash := hash(nodeid) & 0x3ff
+	glog.V(0).Infof("use snowflake seq id generator, nodeid:%s hex_of_nodeid: %x", nodeid, nodeid_hash)
+	node, err := snowflake.NewNode(int64(nodeid_hash))
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+
+	sequencer := &SnowflakeSequencer{node: node}
+	return sequencer, nil
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
+	return uint64(m.node.Generate().Int64())
+}
+
+// ignore setmax as we are snowflake
+func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
+}
+
+// return a new id as no Peek is stored
+func (m *SnowflakeSequencer) Peek() uint64 {
+	return uint64(m.node.Generate().Int64())
+}

weed/server/common.go

@@ -234,12 +234,12 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file
 	}
 }
 
-func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64, httpStatusCode int) error) {
+func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
 	rangeReq := r.Header.Get("Range")
 
 	if rangeReq == "" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
-		if err := writeFn(w, 0, totalSize, 0); err != nil {
+		if err := writeFn(w, 0, totalSize); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -279,7 +279,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
 		w.Header().Set("Content-Range", ra.contentRange(totalSize))
 
-		err = writeFn(w, ra.start, ra.length, http.StatusPartialContent)
+		err = writeFn(w, ra.start, ra.length)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -307,7 +307,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
 				pw.CloseWithError(e)
 				return
 			}
-			if e = writeFn(part, ra.start, ra.length, 0); e != nil {
+			if e = writeFn(part, ra.start, ra.length); e != nil {
 				pw.CloseWithError(e)
 				return
 			}

weed/server/filer_grpc_server_rename.go

@@ -33,8 +33,7 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
 		return nil, fmt.Errorf("%s/%s not found: %v", req.OldDirectory, req.OldName, err)
 	}
 
-	var events MoveEvents
+	moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName)
-	moveErr := fs.moveEntry(ctx, oldParent, oldEntry, newParent, req.NewName, &events)
 	if moveErr != nil {
 		fs.filer.RollbackTransaction(ctx)
 		return nil, fmt.Errorf("%s/%s move error: %v", req.OldDirectory, req.OldName, moveErr)
@@ -48,11 +47,11 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
 	return &filer_pb.AtomicRenameEntryResponse{}, nil
 }
 
-func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
+func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
 
-	if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events, func() error {
+	if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
 		if entry.IsDirectory() {
-			if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
+			if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName); err != nil {
 				return err
 			}
 		}
@@ -64,7 +63,7 @@ func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, e
 	return nil
 }
 
-func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
+func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
 
 	currentDirPath := oldParent.Child(entry.Name())
 	newDirPath := newParent.Child(newName)
@@ -85,7 +84,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
 		for _, item := range entries {
 			lastFileName = item.Name()
 			// println("processing", lastFileName)
-			err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name(), events)
+			err := fs.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
 			if err != nil {
 				return err
 			}
@@ -97,8 +96,7 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
 	return nil
 }
 
-func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, events *MoveEvents,
+func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {
-	moveFolderSubEntries func() error) error {
 
 	oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName)
 
@@ -122,8 +120,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		return createErr
 	}
 
-	events.newEntries = append(events.newEntries, newEntry)
-
 	if moveFolderSubEntries != nil {
 		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
 			return moveChildrenErr
@@ -136,13 +132,6 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		return deleteErr
 	}
 
-	events.oldEntries = append(events.oldEntries, entry)
-
 	return nil
 
 }
-
-type MoveEvents struct {
-	oldEntries []*filer.Entry
-	newEntries []*filer.Entry
-}

weed/server/filer_server.go

@@ -45,22 +45,23 @@ import (
 )
 
 type FilerOption struct {
-	Masters            []string
+	Masters               []string
-	Collection         string
+	Collection            string
-	DefaultReplication string
+	DefaultReplication    string
-	DisableDirListing  bool
+	DisableDirListing     bool
-	MaxMB              int
+	MaxMB                 int
-	DirListingLimit    int
+	DirListingLimit       int
-	DataCenter         string
+	DataCenter            string
-	Rack               string
+	Rack                  string
-	DefaultLevelDbDir  string
+	DefaultLevelDbDir     string
-	DisableHttp        bool
+	DisableHttp           bool
-	Host               string
+	Host                  string
-	Port               uint32
+	Port                  uint32
-	recursiveDelete    bool
+	recursiveDelete       bool
-	Cipher             bool
+	Cipher                bool
-	SaveToFilerLimit   int
+	SaveToFilerLimit      int64
-	Filers             []string
+	Filers                []string
+	ConcurrentUploadLimit int64
 }
 
 type FilerServer struct {
@@ -79,14 +80,18 @@ type FilerServer struct {
 
 	brokers     map[string]map[string]bool
 	brokersLock sync.Mutex
+
+	inFlightDataSize      int64
+	inFlightDataLimitCond *sync.Cond
 }
 
 func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) {
 
 	fs = &FilerServer{
-		option:         option,
+		option:                option,
-		grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
+		grpcDialOption:        security.LoadClientTLS(util.GetViper(), "grpc.filer"),
-		brokers:        make(map[string]map[string]bool),
+		brokers:               make(map[string]map[string]bool),
+		inFlightDataLimitCond: sync.NewCond(new(sync.Mutex)),
 	}
 	fs.listenersCond = sync.NewCond(&fs.listenersLock)
 

weed/server/filer_server_handlers.go

@@ -4,6 +4,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"net/http"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/stats"
@@ -47,18 +48,34 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
 			fs.DeleteHandler(w, r)
 		}
 		stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
-	case "PUT":
+	case "POST", "PUT":
-		stats.FilerRequestCounter.WithLabelValues("put").Inc()
+
-		if _, ok := r.URL.Query()["tagging"]; ok {
+		// wait until in flight data is less than the limit
-			fs.PutTaggingHandler(w, r)
+		contentLength := getContentLength(r)
-		} else {
+		fs.inFlightDataLimitCond.L.Lock()
-			fs.PostHandler(w, r)
+		for atomic.LoadInt64(&fs.inFlightDataSize) > fs.option.ConcurrentUploadLimit {
+			fs.inFlightDataLimitCond.Wait()
+		}
+		atomic.AddInt64(&fs.inFlightDataSize, contentLength)
+		fs.inFlightDataLimitCond.L.Unlock()
+		defer func() {
+			atomic.AddInt64(&fs.inFlightDataSize, -contentLength)
+			fs.inFlightDataLimitCond.Signal()
+		}()
+
+		if r.Method == "PUT" {
+			stats.FilerRequestCounter.WithLabelValues("put").Inc()
+			if _, ok := r.URL.Query()["tagging"]; ok {
+				fs.PutTaggingHandler(w, r)
+			} else {
+				fs.PostHandler(w, r, contentLength)
+			}
+			stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
+		} else { // method == "POST"
+			stats.FilerRequestCounter.WithLabelValues("post").Inc()
+			fs.PostHandler(w, r, contentLength)
+			stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
 		}
-		stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
-	case "POST":
-		stats.FilerRequestCounter.WithLabelValues("post").Inc()
-		fs.PostHandler(w, r)
-		stats.FilerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
 	case "OPTIONS":
 		stats.FilerRequestCounter.WithLabelValues("options").Inc()
 		OptionsHandler(w, r, false)

weed/server/filer_server_handlers_read.go

@@ -131,6 +131,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
+		processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
+			return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true)
+		})
 		return
 	}
 
@@ -150,10 +153,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		}
 	}
 
-	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
+	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
-		if httpStatusCode != 0 {
-			w.WriteHeader(httpStatusCode)
-		}
 		if offset+size <= int64(len(entry.Content)) {
 			_, err := writer.Write(entry.Content[offset : offset+size])
 			if err != nil {
@@ -161,7 +161,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 			}
 			return err
 		}
-		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size)
+		return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false)
 	})
 
 }

weed/server/filer_server_handlers_write.go

@@ -52,7 +52,7 @@ func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, u
 	return
 }
 
-func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
+func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, contentLength int64) {
 
 	ctx := context.Background()
 
@@ -66,7 +66,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		query.Get("rack"),
 	)
 
-	fs.autoChunk(ctx, w, r, so)
+	fs.autoChunk(ctx, w, r, contentLength, so)
 	util.CloseRequest(r)
 
 }

weed/server/filer_server_handlers_write_autochunk.go

@@ -2,11 +2,8 @@ package weed_server
 
 import (
 	"context"
-	"crypto/md5"
 	"fmt"
-	"hash"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -19,13 +16,12 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
-	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
+func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, contentLength int64, so *operation.StorageOption) {
 
 	// autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
 	query := r.URL.Query()
@@ -51,14 +47,16 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 		if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
 			reply, err = fs.mkdir(ctx, w, r)
 		} else {
-			reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
+			reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, contentLength, so)
 		}
 	} else {
-		reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
+		reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, contentLength, so)
 	}
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "read input:") {
 			writeJsonError(w, r, 499, err)
+		} else if strings.HasSuffix(err.Error(), "is a file") {
+			writeJsonError(w, r, http.StatusConflict, err)
 		} else {
 			writeJsonError(w, r, http.StatusInternalServerError, err)
 		}
@@ -70,7 +68,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 	}
 }
 
-func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
+func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
 
 	multipartReader, multipartReaderErr := r.MultipartReader()
 	if multipartReaderErr != nil {
@@ -91,7 +89,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
 		contentType = ""
 	}
 
-	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
+	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, contentLength, so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -102,7 +100,7 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
 	return
 }
 
-func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
+func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, contentLength int64, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
 
 	fileName := path.Base(r.URL.Path)
 	contentType := r.Header.Get("Content-Type")
@@ -110,7 +108,7 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter
 		contentType = ""
 	}
 
-	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
+	fileChunks, md5Hash, chunkOffset, err, smallContent := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, contentLength, so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -212,7 +210,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 		entry.Extended = make(map[string][]byte)
 	}
 
-	fs.saveAmzMetaData(r, entry)
+	SaveAmzMetaData(r, entry.Extended, false)
 
 	for k, v := range r.Header {
 		if len(v) > 0 && strings.HasPrefix(k, needle.PairNamePrefix) {
@@ -229,92 +227,6 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 	return filerResult, replyerr
 }
 
-func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
-	var fileChunks []*filer_pb.FileChunk
-
-	md5Hash := md5.New()
-	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
-
-	chunkOffset := int64(0)
-	var smallContent []byte
-
-	for {
-		limitedReader := io.LimitReader(partReader, int64(chunkSize))
-
-		data, err := ioutil.ReadAll(limitedReader)
-		if err != nil {
-			return nil, nil, 0, err, nil
-		}
-		if chunkOffset == 0 && !isAppend(r) {
-			if len(data) < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && len(data) < 4*1024 {
-				smallContent = data
-				chunkOffset += int64(len(data))
-				break
-			}
-		}
-		dataReader := util.NewBytesReader(data)
-
-		// retry to assign a different file id
-		var fileId, urlLocation string
-		var auth security.EncodedJwt
-		var assignErr, uploadErr error
-		var uploadResult *operation.UploadResult
-		for i := 0; i < 3; i++ {
-			// assign one file id for one chunk
-			fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so)
-			if assignErr != nil {
-				return nil, nil, 0, assignErr, nil
-			}
-
-			// upload the chunk to the volume server
-			uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth)
-			if uploadErr != nil {
-				time.Sleep(251 * time.Millisecond)
-				continue
-			}
-			break
-		}
-		if uploadErr != nil {
-			return nil, nil, 0, uploadErr, nil
-		}
-
-		// if last chunk exhausted the reader exactly at the border
-		if uploadResult.Size == 0 {
-			break
-		}
-
-		// Save to chunk manifest structure
-		fileChunks = append(fileChunks, uploadResult.ToPbFileChunk(fileId, chunkOffset))
-
-		glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), fileId, chunkOffset, chunkOffset+int64(uploadResult.Size))
-
-		// reset variables for the next chunk
-		chunkOffset = chunkOffset + int64(uploadResult.Size)
-
-		// if last chunk was not at full chunk size, but already exhausted the reader
-		if int64(uploadResult.Size) < int64(chunkSize) {
-			break
-		}
-	}
-
-	return fileChunks, md5Hash, chunkOffset, nil, smallContent
-}
-
-func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
-
-	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
-	start := time.Now()
-	defer func() {
-		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
-	}()
-
-	uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
-	if uploadResult != nil && uploadResult.RetryCount > 0 {
-		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
-	}
-	return uploadResult, err, data
-}
-
 func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
 
 	return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
@@ -383,17 +295,24 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http
 	return filerResult, replyerr
 }
 
-func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
+func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte) {
+
+	metadata = make(map[string][]byte)
+	if !isReplace {
+		for k, v := range existing {
+			metadata[k] = v
+		}
+	}
 
 	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
-		entry.Extended[xhttp.AmzStorageClass] = []byte(sc)
+		metadata[xhttp.AmzStorageClass] = []byte(sc)
 	}
 
 	if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
 		for _, v := range strings.Split(tags, "&") {
 			tag := strings.Split(v, "=")
 			if len(tag) == 2 {
-				entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
+				metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
 			}
 		}
 	}
@@ -401,8 +320,11 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) {
 	for header, values := range r.Header {
 		if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
 			for _, value := range values {
-				entry.Extended[header] = []byte(value)
+				metadata[header] = []byte(value)
 			}
 		}
 	}
+
+	return
+
 }

weed/server/filer_server_handlers_write_upload.go

@@ -0,0 +1,168 @@
+package weed_server
+
+import (
+	"crypto/md5"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/stats"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	limitedUploadProcessor = util.NewLimitedOutOfOrderProcessor(int32(runtime.NumCPU()))
+)
+
+func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, dataSize int64, err error, smallContent []byte) {
+
+	md5Hash = md5.New()
+	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
+
+	// save small content directly
+	if !isAppend(r) && ((0 < contentLength && contentLength < fs.option.SaveToFilerLimit) || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && contentLength < 4*1024) {
+		smallContent, err = ioutil.ReadAll(partReader)
+		dataSize = int64(len(smallContent))
+		return
+	}
+
+	resultsChan := make(chan *ChunkCreationResult, 2)
+
+	var waitForAllData sync.WaitGroup
+	waitForAllData.Add(1)
+	go func() {
+		// process upload results
+		defer waitForAllData.Done()
+		for result := range resultsChan {
+			if result.err != nil {
+				err = result.err
+				continue
+			}
+
+			// Save to chunk manifest structure
+			fileChunks = append(fileChunks, result.chunk)
+		}
+	}()
+
+	var lock sync.Mutex
+	readOffset := int64(0)
+	var wg sync.WaitGroup
+
+	for err == nil {
+
+		wg.Add(1)
+		request := func() {
+			defer wg.Done()
+
+			var localOffset int64
+			// read from the input
+			lock.Lock()
+			localOffset = readOffset
+			limitedReader := io.LimitReader(partReader, int64(chunkSize))
+			data, readErr := ioutil.ReadAll(limitedReader)
+			readOffset += int64(len(data))
+			lock.Unlock()
+			// handle read errors
+			if readErr != nil {
+				if readErr != io.EOF {
+					if err == nil {
+						err = readErr
+					}
+					resultsChan <- &ChunkCreationResult{
+						err: readErr,
+					}
+				}
+				return
+			}
+			if len(data) == 0 {
+				readErr = io.EOF
+				return
+			}
+
+			// upload
+			dataReader := util.NewBytesReader(data)
+			fileId, uploadResult, uploadErr := fs.doCreateChunk(w, r, so, dataReader, fileName, contentType)
+			if uploadErr != nil {
+				if err == nil {
+					err = uploadErr
+				}
+				resultsChan <- &ChunkCreationResult{
+					err: uploadErr,
+				}
+				return
+			}
+
+			glog.V(4).Infof("uploaded %s to %s [%d,%d)", fileName, fileId, localOffset, localOffset+int64(uploadResult.Size))
+
+			// send back uploaded file chunk
+			resultsChan <- &ChunkCreationResult{
+				chunk: uploadResult.ToPbFileChunk(fileId, localOffset),
+			}
+
+		}
+		limitedUploadProcessor.Execute(request)
+	}
+
+	go func() {
+		wg.Wait()
+		close(resultsChan)
+	}()
+
+	waitForAllData.Wait()
+
+	return fileChunks, md5Hash, readOffset, err, nil
+}
+
+type ChunkCreationResult struct {
+	chunk *filer_pb.FileChunk
+	err   error
+}
+
+func (fs *FilerServer) doCreateChunk(w http.ResponseWriter, r *http.Request, so *operation.StorageOption, dataReader *util.BytesReader, fileName string, contentType string) (string, *operation.UploadResult, error) {
+	// retry to assign a different file id
+	var fileId, urlLocation string
+	var auth security.EncodedJwt
+	var assignErr, uploadErr error
+	var uploadResult *operation.UploadResult
+	for i := 0; i < 3; i++ {
+		// assign one file id for one chunk
+		fileId, urlLocation, auth, assignErr = fs.assignNewFileInfo(so)
+		if assignErr != nil {
+			return "", nil, assignErr
+		}
+
+		// upload the chunk to the volume server
+		uploadResult, uploadErr, _ = fs.doUpload(urlLocation, w, r, dataReader, fileName, contentType, nil, auth)
+		if uploadErr != nil {
+			time.Sleep(251 * time.Millisecond)
+			continue
+		}
+		break
+	}
+	return fileId, uploadResult, uploadErr
+}
+
+func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
+
+	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
+	}()
+
+	uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
+	if uploadResult != nil && uploadResult.RetryCount > 0 {
+		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
+	}
+	return uploadResult, err, data
+}

weed/server/master_grpc_server.go

@@ -80,10 +80,14 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 		dn.AdjustMaxVolumeCounts(heartbeat.MaxVolumeCounts)
 
 		glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
+		var dataCenter string
+		if dc := dn.GetDataCenter(); dc != nil {
+			dataCenter = string(dc.Id())
+		}
 		message := &master_pb.VolumeLocation{
 			Url:        dn.Url(),
 			PublicUrl:  dn.PublicUrl,
-			DataCenter: string(dn.GetDataCenter().Id()),
+			DataCenter: dataCenter,
 		}
 		if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
 			// process delta volume ids if exists for fast volume id updates

weed/server/master_server.go

@@ -277,6 +277,13 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
 			glog.Error(err)
 			seq = nil
 		}
+	case "snowflake":
+		var err error
+		seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
+		if err != nil {
+			glog.Error(err)
+			seq = nil
+		}
 	default:
 		seq = sequence.NewMemorySequencer()
 	}

weed/server/volume_grpc_read_write.go

@@ -0,0 +1,38 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_pb.ReadNeedleBlobRequest) (resp *volume_server_pb.ReadNeedleBlobResponse, err error) {
+	resp = &volume_server_pb.ReadNeedleBlobResponse{}
+	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
+	if v == nil {
+		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+	}
+
+	resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size))
+	if err != nil {
+		return nil, fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err)
+	}
+
+	return resp, nil
+}
+
+func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_pb.WriteNeedleBlobRequest) (resp *volume_server_pb.WriteNeedleBlobResponse, err error) {
+	resp = &volume_server_pb.WriteNeedleBlobResponse{}
+	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
+	if v == nil {
+		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+	}
+
+	if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil {
+		return nil, fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err)
+	}
+
+	return resp, nil
+}

weed/server/volume_server.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"net/http"
+	"sync"
 
 	"google.golang.org/grpc"
 
@@ -34,6 +35,10 @@ type VolumeServer struct {
 	fileSizeLimitBytes      int64
 	isHeartbeating          bool
 	stopChan                chan bool
+
+	inFlightDataSize      int64
+	inFlightDataLimitCond *sync.Cond
+	concurrentUploadLimit int64
 }
 
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
@@ -48,6 +53,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	readRedirect bool,
 	compactionMBPerSecond int,
 	fileSizeLimitMB int,
+	concurrentUploadLimit int64,
 ) *VolumeServer {
 
 	v := util.GetViper()
@@ -72,6 +78,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 		fileSizeLimitBytes:      int64(fileSizeLimitMB) * 1024 * 1024,
 		isHeartbeating:          true,
 		stopChan:                make(chan bool),
+		inFlightDataLimitCond:   sync.NewCond(new(sync.Mutex)),
+		concurrentUploadLimit:   concurrentUploadLimit,
 	}
 	vs.SeedMasterNodes = masterNodes
 

weed/server/volume_server_handlers.go

@@ -2,7 +2,9 @@ package weed_server
 
 import (
 	"net/http"
+	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/chrislusf/seaweedfs/weed/util"
 
@@ -40,8 +42,24 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 		stats.DeleteRequest()
 		vs.guard.WhiteList(vs.DeleteHandler)(w, r)
 	case "PUT", "POST":
+
+		// wait until in flight data is less than the limit
+		contentLength := getContentLength(r)
+		vs.inFlightDataLimitCond.L.Lock()
+		for atomic.LoadInt64(&vs.inFlightDataSize) > vs.concurrentUploadLimit {
+			vs.inFlightDataLimitCond.Wait()
+		}
+		atomic.AddInt64(&vs.inFlightDataSize, contentLength)
+		vs.inFlightDataLimitCond.L.Unlock()
+		defer func() {
+			atomic.AddInt64(&vs.inFlightDataSize, -contentLength)
+			vs.inFlightDataLimitCond.Signal()
+		}()
+
+		// processs uploads
 		stats.WriteRequest()
 		vs.guard.WhiteList(vs.PostHandler)(w, r)
+
 	case "OPTIONS":
 		stats.ReadRequest()
 		w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
@@ -49,6 +67,18 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 	}
 }
 
+func getContentLength(r *http.Request) int64 {
+	contentLength := r.Header.Get("Content-Length")
+	if contentLength != "" {
+		length, err := strconv.ParseInt(contentLength, 10, 64)
+		if err != nil {
+			return 0
+		}
+		return length
+	}
+	return 0
+}
+
 func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
 	if r.Header.Get("Origin") != "" {

weed/server/volume_server_handlers_read.go

@@ -27,7 +27,7 @@ var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
 
 func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 
-	// println(r.Method + " " + r.URL.Path)
+	glog.V(9).Info(r.Method + " " + r.URL.Path + " " + r.Header.Get("Range"))
 
 	stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
 	start := time.Now()
@@ -261,13 +261,10 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re
 		return nil
 	}
 
-	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64, httpStatusCode int) error {
+	processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error {
 		if _, e = rs.Seek(offset, 0); e != nil {
 			return e
 		}
-		if httpStatusCode != 0 {
-			w.WriteHeader(httpStatusCode)
-		}
 		_, e = io.CopyN(writer, rs, size)
 		return e
 	})

weed/shell/command_fs_cat.go

@@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
 			return err
 		}
 
-		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
 
 	})
 

weed/shell/command_fs_mv.go

@@ -36,6 +36,10 @@ func (c *commandFsMv) Help() string {
 
 func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
 
+	if len(args) != 2 {
+		return fmt.Errorf("need to have 2 arguments")
+	}
+
 	sourcePath, err := commandEnv.parseUrl(args[0])
 	if err != nil {
 		return err

weed/shell/command_s3_clean_uploads.go

@@ -54,7 +54,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
 		return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
 	}
 
-	for _, bucket:= range buckets {
+	for _, bucket := range buckets {
 		c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo)
 	}
 
@@ -63,7 +63,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
 }
 
 func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error {
-	uploadsDir := filerBucketsPath+"/"+bucket+"/.uploads"
+	uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
 	var staleUploads []string
 	now := time.Now()
 	err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
@@ -77,8 +77,8 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
 		return fmt.Errorf("list uploads under %v: %v", uploadsDir, err)
 	}
 
-	for _, staleUpload:= range staleUploads {
+	for _, staleUpload := range staleUploads {
-		deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true",commandEnv.option.FilerHost, commandEnv.option.FilerPort,uploadsDir, staleUpload)
+		deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerHost, commandEnv.option.FilerPort, uploadsDir, staleUpload)
 		fmt.Fprintf(writer, "purge %s\n", deleteUrl)
 
 		err = util.Delete(deleteUrl, "")
@@ -89,4 +89,4 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
 
 	return nil
 
-}
+}

weed/shell/command_volume_balance.go

@@ -393,9 +393,8 @@ func adjustAfterMove(v *master_pb.VolumeInformationMessage, volumeReplicas map[u
 		if replica.location.dataNode.Id == fullNode.info.Id &&
 			replica.location.rack == fullNode.rack &&
 			replica.location.dc == fullNode.dc {
-			replica.location.dc = emptyNode.dc
+			loc := newLocation(emptyNode.dc, emptyNode.rack, emptyNode.info)
-			replica.location.rack = emptyNode.rack
+			replica.location = &loc
-			replica.location.dataNode = emptyNode.info
 			return
 		}
 	}

weed/shell/command_volume_balance_test.go

@@ -169,3 +169,15 @@ func TestIsGoodMove(t *testing.T) {
 	}
 
 }
+
+func TestBalance(t *testing.T) {
+	topologyInfo := parseOutput(topoData)
+	volumeServers := collectVolumeServersByDc(topologyInfo, "")
+	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
+	diskTypes := collectVolumeDiskTypes(topologyInfo)
+
+	if err := balanceVolumeServers(nil, diskTypes, volumeReplicas, volumeServers, 30*1024*1024*1024, "ALL_COLLECTIONS", false); err != nil {
+		t.Errorf("balance: %v", err)
+	}
+
+}

weed/shell/command_volume_check_disk.go

@@ -0,0 +1,257 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+	"io"
+	"math"
+	"sort"
+)
+
+func init() {
+	Commands = append(Commands, &commandVolumeCheckDisk{})
+}
+
+type commandVolumeCheckDisk struct {
+	env *CommandEnv
+}
+
+func (c *commandVolumeCheckDisk) Name() string {
+	return "volume.check.disk"
+}
+
+func (c *commandVolumeCheckDisk) Help() string {
+	return `check all replicated volumes to find and fix inconsistencies
+
+	How it works:
+	
+	find all volumes that are replicated
+	  for each volume id, if there are more than 2 replicas, find one pair with the largest 2 in file count.
+      for the pair volume A and B
+        append entries in A and not in B to B
+        append entries in B and not in A to A
+
+`
+}
+
+func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if err = commandEnv.confirmIsLocked(); err != nil {
+		return
+	}
+
+	fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	slowMode := fsckCommand.Bool("slow", false, "slow mode checks all replicas even file counts are the same")
+	verbose := fsckCommand.Bool("v", false, "verbose mode")
+	applyChanges := fsckCommand.Bool("force", false, "apply the fix")
+	nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit")
+	if err = fsckCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	c.env = commandEnv
+
+	// collect topology information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv)
+	if err != nil {
+		return err
+	}
+	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
+
+	// pick 1 pairs of volume replica
+	fileCount := func(replica *VolumeReplica) uint64 {
+		return replica.info.FileCount - replica.info.DeleteCount
+	}
+	aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
+	defer aDB.Close()
+	defer bDB.Close()
+
+	for _, replicas := range volumeReplicas {
+		sort.Slice(replicas, func(i, j int) bool {
+			return fileCount(replicas[i]) > fileCount(replicas[j])
+		})
+		for len(replicas) >= 2 {
+			a, b := replicas[0], replicas[1]
+			if !*slowMode {
+				if fileCount(a) == fileCount(b) {
+					replicas = replicas[1:]
+					continue
+				}
+			}
+			if a.info.ReadOnly || b.info.ReadOnly {
+				fmt.Fprintf(writer, "skipping readonly volume %d on %s and %s\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
+				continue
+			}
+
+			// reset index db
+			aDB.Close()
+			bDB.Close()
+			aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb()
+
+			// read index db
+			if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil {
+				return err
+			}
+			if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil {
+				return err
+			}
+
+			// find and make up the differnces
+			if err := c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
+				return err
+			}
+			if err := c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
+				return err
+			}
+			replicas = replicas[1:]
+		}
+	}
+
+	return nil
+}
+
+func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) error {
+
+	// find missing keys
+	// hash join, can be more efficient
+	var missingNeedles []needle_map.NeedleValue
+	var counter int
+	subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
+		counter++
+		if _, found := minuend.Get(value.Key); !found {
+			missingNeedles = append(missingNeedles, value)
+		}
+		return nil
+	})
+
+	fmt.Fprintf(writer, "volume %d %s has %d entries, %s missed %d entries\n", source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles))
+
+	if counter == 0 || len(missingNeedles) == 0 {
+		return nil
+	}
+
+	missingNeedlesFraction := float64(len(missingNeedles)) / float64(counter)
+	if missingNeedlesFraction > nonRepairThreshold {
+		return fmt.Errorf(
+			"failed to start repair volume %d, percentage of missing keys is greater than the threshold: %.2f > %.2f",
+			source.info.Id, missingNeedlesFraction, nonRepairThreshold)
+	}
+
+	for _, needleValue := range missingNeedles {
+
+		needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue)
+		if err != nil {
+			return err
+		}
+
+		if !applyChanges {
+			continue
+		}
+
+		if verbose {
+			fmt.Fprintf(writer, "read %d,%x %s => %s \n", source.info.Id, needleValue.Key, source.location.dataNode.Id, target.location.dataNode.Id)
+		}
+
+		if err := c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) {
+
+	err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{
+			VolumeId: volumeId,
+			NeedleId: uint64(needleValue.Key),
+			Offset:   needleValue.Offset.ToActualOffset(),
+			Size:     int32(needleValue.Size),
+		})
+		if err != nil {
+			return err
+		}
+		needleBlob = resp.NeedleBlob
+		return nil
+	})
+	return
+}
+
+func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error {
+
+	return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
+			VolumeId:   volumeId,
+			NeedleId:   uint64(needleValue.Key),
+			Size:       int32(needleValue.Size),
+			NeedleBlob: needleBlob,
+		})
+		return err
+	})
+
+}
+
+func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error {
+
+	var buf bytes.Buffer
+	if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil {
+		return err
+	}
+
+	if verbose {
+		fmt.Fprintf(writer, "load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
+	}
+
+	return db.LoadFromReaderAt(bytes.NewReader(buf.Bytes()))
+
+}
+
+func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error {
+
+	return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+		ext := ".idx"
+
+		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
+			VolumeId:                 volumeId,
+			Ext:                      ".idx",
+			CompactionRevision:       math.MaxUint32,
+			StopOffset:               math.MaxInt64,
+			Collection:               collection,
+			IsEcVolume:               false,
+			IgnoreSourceFileNotFound: false,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
+		}
+
+		err = writeToBuffer(copyFileClient, buf)
+		if err != nil {
+			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, volumeServer, err)
+		}
+
+		return nil
+
+	})
+}
+
+func writeToBuffer(client volume_server_pb.VolumeServer_CopyFileClient, buf *bytes.Buffer) error {
+	for {
+		resp, receiveErr := client.Recv()
+		if receiveErr == io.EOF {
+			break
+		}
+		if receiveErr != nil {
+			return fmt.Errorf("receiving: %v", receiveErr)
+		}
+		buf.Write(resp.FileContent)
+	}
+	return nil
+}

weed/shell/command_volume_list_test.go

@@ -0,0 +1,893 @@
+package shell
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/golang/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func TestParsing(t *testing.T) {
+	topo := parseOutput(topoData)
+
+	assert.Equal(t, 5, len(topo.DataCenterInfos))
+
+}
+
+func parseOutput(output string) *master_pb.TopologyInfo {
+	lines := strings.Split(output, "\n")
+	var topo *master_pb.TopologyInfo
+	var dc *master_pb.DataCenterInfo
+	var rack *master_pb.RackInfo
+	var dn *master_pb.DataNodeInfo
+	var disk *master_pb.DiskInfo
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		parts := strings.Split(line, " ")
+		switch parts[0] {
+		case "Topology":
+			if topo == nil {
+				topo = &master_pb.TopologyInfo{}
+			}
+		case "DataCenter":
+			if dc == nil {
+				dc = &master_pb.DataCenterInfo{
+					Id: parts[1],
+				}
+				topo.DataCenterInfos = append(topo.DataCenterInfos, dc)
+			} else {
+				dc = nil
+			}
+		case "Rack":
+			if rack == nil {
+				rack = &master_pb.RackInfo{
+					Id: parts[1],
+				}
+				dc.RackInfos = append(dc.RackInfos, rack)
+			} else {
+				rack = nil
+			}
+		case "DataNode":
+			if dn == nil {
+				dn = &master_pb.DataNodeInfo{
+					Id:        parts[1],
+					DiskInfos: make(map[string]*master_pb.DiskInfo),
+				}
+				rack.DataNodeInfos = append(rack.DataNodeInfos, dn)
+			} else {
+				dn = nil
+			}
+		case "Disk":
+			if disk == nil {
+				diskType := parts[1][:strings.Index(parts[1], "(")]
+				maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
+				maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
+				disk = &master_pb.DiskInfo{
+					Type:           diskType,
+					MaxVolumeCount: uint64(maxVolumeCount),
+				}
+				dn.DiskInfos[types.ToDiskType(diskType).String()] = disk
+			} else {
+				disk = nil
+			}
+		case "volume":
+			volumeLine := line[len("volume "):]
+			volume := &master_pb.VolumeInformationMessage{}
+			proto.UnmarshalText(volumeLine, volume)
+			disk.VolumeInfos = append(disk.VolumeInfos, volume)
+		}
+	}
+
+	return topo
+}
+
+const topoData = `
+Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0)
+  DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0)
+    Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0)
+    Rack DefaultRack total size:0 file_count:0 
+  DataCenter dc1 total size:0 file_count:0 
+  DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0)
+    Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0)
+      DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0)
+        Disk hdd(volume:50/240 active:50 free:190 remote:0)
+          volume id:15  size:1115965064  collection:"collection0"  file_count:83  replica_placement:100  version:3  modified_at_second:1609923671 
+          volume id:21  size:1097631536  collection:"collection0"  file_count:82  delete_count:7  deleted_byte_count:68975485  replica_placement:100  version:3  modified_at_second:1609929578 
+          volume id:22  size:1086828272  collection:"collection0"  file_count:75  replica_placement:100  version:3  modified_at_second:1609930001 
+          volume id:23  size:1076380216  collection:"collection0"  file_count:68  replica_placement:100  version:3  modified_at_second:1609930434 
+          volume id:24  size:1074139776  collection:"collection0"  file_count:90  replica_placement:100  version:3  modified_at_second:1609930909 
+          volume id:25  size:690757512  collection:"collection0"  file_count:38  replica_placement:100  version:3  modified_at_second:1611144216 
+          volume id:27  size:298886792  file_count:1608  replica_placement:100  version:3  modified_at_second:1615632482 
+          volume id:28  size:308919192  file_count:1591  delete_count:1  deleted_byte_count:125280  replica_placement:100  version:3  modified_at_second:1615631762 
+          volume id:29  size:281582680  file_count:1537  replica_placement:100  version:3  modified_at_second:1615629422 
+          volume id:30  size:289466144  file_count:1566  delete_count:1  deleted_byte_count:124972  replica_placement:100  version:3  modified_at_second:1615632422 
+          volume id:31  size:273363256  file_count:1498  replica_placement:100  version:3  modified_at_second:1615631642 
+          volume id:33  size:1130226400  collection:"collection1"  file_count:7322  delete_count:172  deleted_byte_count:45199399  replica_placement:100  version:3  modified_at_second:1615618789 
+          volume id:38  size:1075545744  collection:"collection1"  file_count:13324  delete_count:100  deleted_byte_count:25223906  replica_placement:100  version:3  modified_at_second:1615569830 
+          volume id:51  size:1076796120  collection:"collection1"  file_count:10550  delete_count:39  deleted_byte_count:12723654  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615547786 
+          volume id:52  size:1083529728  collection:"collection1"  file_count:10128  delete_count:32  deleted_byte_count:10608391  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615599195 
+          volume id:54  size:1045022344  collection:"collection1"  file_count:9408  delete_count:30  deleted_byte_count:15132106  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
+          volume id:63  size:956941112  collection:"collection1"  file_count:8271  delete_count:32  deleted_byte_count:15876189  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632036 
+          volume id:69  size:869213648  collection:"collection1"  file_count:7293  delete_count:102  deleted_byte_count:30643207  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630534 
+          volume id:74  size:957046128  collection:"collection1"  file_count:6982  delete_count:258  deleted_byte_count:73054259  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631460 
+          volume id:80  size:827912928  collection:"collection1"  file_count:6914  delete_count:17  deleted_byte_count:5689635  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631157 
+          volume id:84  size:873121856  collection:"collection1"  file_count:8200  delete_count:13  deleted_byte_count:3131676  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631161 
+          volume id:85  size:1023869320  collection:"collection1"  file_count:7788  delete_count:234  deleted_byte_count:78037967  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631723 
+          volume id:97  size:1053112992  collection:"collection1"  file_count:6789  delete_count:50  deleted_byte_count:38894001  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631193 
+          volume id:98  size:1077836440  collection:"collection1"  file_count:7605  delete_count:202  deleted_byte_count:73180379  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615523691 
+          volume id:105  size:1073996824  collection:"collection1"  file_count:6872  delete_count:20  deleted_byte_count:14482293  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615499757 
+          volume id:106  size:1075458664  collection:"collection1"  file_count:7182  delete_count:307  deleted_byte_count:69349053  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615598137 
+          volume id:112  size:1076392512  collection:"collection1"  file_count:8291  delete_count:156  deleted_byte_count:74120183  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569823 
+          volume id:116  size:1074489504  collection:"collection1"  file_count:9981  delete_count:174  deleted_byte_count:53998777  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615611565 
+          volume id:119  size:1075940104  collection:"collection1"  file_count:9003  delete_count:12  deleted_byte_count:9128155  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573878 
+          volume id:128  size:1074874632  collection:"collection1"  file_count:9821  delete_count:148  deleted_byte_count:43633334  replica_placement:100  version:3  modified_at_second:1615602670 
+          volume id:133  size:1075952760  collection:"collection1"  file_count:9538  delete_count:74  deleted_byte_count:19558008  replica_placement:100  version:3  modified_at_second:1615584779 
+          volume id:136  size:1074433552  collection:"collection1"  file_count:9593  delete_count:72  deleted_byte_count:26912512  replica_placement:100  version:3  modified_at_second:1615376036 
+          volume id:138  size:1074465744  collection:"collection1"  file_count:10120  delete_count:55  deleted_byte_count:15875438  replica_placement:100  version:3  modified_at_second:1615572231 
+          volume id:140  size:1076203744  collection:"collection1"  file_count:11219  delete_count:57  deleted_byte_count:19864498  replica_placement:100  version:3  modified_at_second:1615571947 
+          volume id:144  size:1074549720  collection:"collection1"  file_count:8780  delete_count:50  deleted_byte_count:52475146  replica_placement:100  version:3  modified_at_second:1615573451 
+          volume id:161  size:1077397192  collection:"collection1"  file_count:9988  delete_count:28  deleted_byte_count:12509164  replica_placement:100  version:3  modified_at_second:1615631452 
+          volume id:173  size:1074154704  collection:"collection1"  file_count:30884  delete_count:34  deleted_byte_count:2578509  replica_placement:100  version:3  modified_at_second:1615591904 
+          volume id:174  size:1073824232  collection:"collection1"  file_count:30689  delete_count:36  deleted_byte_count:2160116  replica_placement:100  version:3  modified_at_second:1615598914 
+          volume id:197  size:1075423240  collection:"collection1"  file_count:16473  delete_count:15  deleted_byte_count:12552442  replica_placement:100  version:3  modified_at_second:1615485254 
+          volume id:219  size:1092298904  collection:"collection1"  file_count:3193  delete_count:17  deleted_byte_count:2047576  replica_placement:100  version:3  modified_at_second:1615579316 
+          volume id:263  size:1077167352  collection:"collection2"  file_count:20227  delete_count:4  deleted_byte_count:97887  replica_placement:100  version:3  modified_at_second:1614871567 
+          volume id:272  size:1076146040  collection:"collection2"  file_count:21034  delete_count:2  deleted_byte_count:216564  replica_placement:100  version:3  modified_at_second:1614884139 
+          volume id:291  size:1076256760  collection:"collection2"  file_count:28301  delete_count:5  deleted_byte_count:116027  replica_placement:100  version:3  modified_at_second:1614904924 
+          volume id:299  size:1075147824  collection:"collection2"  file_count:22927  delete_count:4  deleted_byte_count:345569  replica_placement:100  version:3  modified_at_second:1614918454 
+          volume id:301  size:1074655600  collection:"collection2"  file_count:22543  delete_count:6  deleted_byte_count:136968  replica_placement:100  version:3  modified_at_second:1614918378 
+          volume id:302  size:1077559792  collection:"collection2"  file_count:23124  delete_count:7  deleted_byte_count:293111  replica_placement:100  version:3  modified_at_second:1614925500 
+          volume id:339  size:1078402392  collection:"collection2"  file_count:22309  replica_placement:100  version:3  modified_at_second:1614969996 
+          volume id:345  size:1074560760  collection:"collection2"  file_count:22117  delete_count:2  deleted_byte_count:373286  replica_placement:100  version:3  modified_at_second:1614977458 
+          volume id:355  size:1075239792  collection:"collection2"  file_count:22244  delete_count:1  deleted_byte_count:23282  replica_placement:100  version:3  modified_at_second:1614992157 
+          volume id:373  size:1080928000  collection:"collection2"  file_count:22617  delete_count:4  deleted_byte_count:91849  replica_placement:100  version:3  modified_at_second:1615016877 
+        Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
+      DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
+    Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 
+    Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0)
+      DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0)
+        Disk hdd(volume:36/190 active:36 free:154 remote:0)
+          volume id:2  size:289228560  file_count:1640  delete_count:4  deleted_byte_count:480564  replica_placement:100  version:3  compact_revision:6  modified_at_second:1615630622 
+          volume id:3  size:308743136  file_count:1638  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632242 
+          volume id:4  size:285986968  file_count:1641  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632302 
+          volume id:6  size:302411024  file_count:1604  delete_count:2  deleted_byte_count:274587  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631402 
+          volume id:7  size:1924728  collection:"collection4"  file_count:15  replica_placement:100  version:3  modified_at_second:1609331040 
+          volume id:9  size:77337416  collection:"collection3"  file_count:58  replica_placement:100  version:3  ttl:772  modified_at_second:1615513762 
+          volume id:10  size:1212784656  collection:"collection0"  file_count:58  replica_placement:100  version:3  modified_at_second:1609814550 
+          volume id:12  size:1110923848  collection:"collection0"  file_count:45  replica_placement:100  version:3  modified_at_second:1609819732 
+          volume id:13  size:1184910656  collection:"collection0"  file_count:47  replica_placement:100  version:3  modified_at_second:1609827837 
+          volume id:14  size:1107475720  collection:"collection0"  file_count:80  delete_count:3  deleted_byte_count:6870  replica_placement:100  version:3  modified_at_second:1612956980 
+          volume id:16  size:1113666104  collection:"collection0"  file_count:73  delete_count:5  deleted_byte_count:6318  replica_placement:100  version:3  modified_at_second:1612957007 
+          volume id:17  size:1095115800  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:7099  replica_placement:100  version:3  modified_at_second:1612957000 
+          volume id:21  size:1097631664  collection:"collection0"  file_count:82  delete_count:11  deleted_byte_count:68985100  replica_placement:100  version:3  modified_at_second:1612957007 
+          volume id:56  size:1001897616  collection:"collection1"  file_count:8762  delete_count:37  deleted_byte_count:65375405  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632014 
+          volume id:81  size:880693104  collection:"collection1"  file_count:7481  delete_count:236  deleted_byte_count:80386421  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631396 
+          volume id:104  size:1076383624  collection:"collection1"  file_count:7663  delete_count:184  deleted_byte_count:100728071  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615602658 
+          volume id:107  size:1073811840  collection:"collection1"  file_count:7436  delete_count:168  deleted_byte_count:57747484  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615293569 
+          volume id:113  size:1076709184  collection:"collection1"  file_count:9355  delete_count:177  deleted_byte_count:59796765  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569822 
+          volume id:139  size:1074163936  collection:"collection1"  file_count:9315  delete_count:42  deleted_byte_count:10630966  replica_placement:100  version:3  modified_at_second:1615571946 
+          volume id:151  size:1098659752  collection:"collection1"  file_count:10808  delete_count:24  deleted_byte_count:7088102  replica_placement:100  version:3  modified_at_second:1615586389 
+          volume id:155  size:1075140688  collection:"collection1"  file_count:10882  delete_count:32  deleted_byte_count:9076141  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:167  size:1073958176  collection:"collection1"  file_count:25229  delete_count:48  deleted_byte_count:25871565  replica_placement:100  version:3  modified_at_second:1615602669 
+          volume id:177  size:1074120216  collection:"collection1"  file_count:22293  delete_count:16  deleted_byte_count:3803952  replica_placement:100  version:3  modified_at_second:1615516892 
+          volume id:179  size:1074313920  collection:"collection1"  file_count:21829  delete_count:24  deleted_byte_count:45552859  replica_placement:100  version:3  modified_at_second:1615580308 
+          volume id:182  size:1076131280  collection:"collection1"  file_count:31987  delete_count:21  deleted_byte_count:1452346  replica_placement:100  version:3  modified_at_second:1615568922 
+          volume id:215  size:1068268216  collection:"collection1"  file_count:2813  delete_count:10  deleted_byte_count:5676795  replica_placement:100  version:3  modified_at_second:1615586386 
+          volume id:217  size:1075381872  collection:"collection1"  file_count:3331  delete_count:14  deleted_byte_count:2009141  replica_placement:100  version:3  modified_at_second:1615401638 
+          volume id:283  size:1080178944  collection:"collection2"  file_count:19462  delete_count:7  deleted_byte_count:660407  replica_placement:100  version:3  modified_at_second:1614896626 
+          volume id:303  size:1075944504  collection:"collection2"  file_count:22541  delete_count:2  deleted_byte_count:13617  replica_placement:100  version:3  modified_at_second:1614925431 
+          volume id:309  size:1075178624  collection:"collection2"  file_count:22692  delete_count:3  deleted_byte_count:171124  replica_placement:100  version:3  modified_at_second:1614931409 
+          volume id:323  size:1074608200  collection:"collection2"  file_count:21605  delete_count:4  deleted_byte_count:172090  replica_placement:100  version:3  modified_at_second:1614950526 
+          volume id:344  size:1075035448  collection:"collection2"  file_count:21765  delete_count:1  deleted_byte_count:24623  replica_placement:100  version:3  modified_at_second:1614977465 
+          volume id:347  size:1075145496  collection:"collection2"  file_count:22178  delete_count:1  deleted_byte_count:79392  replica_placement:100  version:3  modified_at_second:1614984727 
+          volume id:357  size:1074276208  collection:"collection2"  file_count:23137  delete_count:4  deleted_byte_count:188487  replica_placement:100  version:3  modified_at_second:1614998792 
+          volume id:380  size:1010760456  collection:"collection2"  file_count:14921  delete_count:6  deleted_byte_count:65678  replica_placement:100  version:3  modified_at_second:1615632322 
+          volume id:381  size:939292792  collection:"collection2"  file_count:14619  delete_count:2  deleted_byte_count:5119  replica_placement:100  version:3  modified_at_second:1615632324 
+        Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
+      DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
+    Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 
+  DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 
+  DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0)
+    Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0)
+      DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0)
+        Disk hdd(volume:108/1850 active:108 free:1742 remote:0)
+          volume id:1  size:284685936  file_count:1557  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632062 
+          volume id:32  size:281390512  file_count:1496  delete_count:6  deleted_byte_count:546403  replica_placement:100  version:3  modified_at_second:1615632362 
+          volume id:47  size:444599784  collection:"collection1"  file_count:709  delete_count:19  deleted_byte_count:11913451  replica_placement:100  version:3  modified_at_second:1615632397 
+          volume id:49  size:1078775288  collection:"collection1"  file_count:9636  delete_count:22  deleted_byte_count:5625976  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630446 
+          volume id:68  size:898630584  collection:"collection1"  file_count:6934  delete_count:95  deleted_byte_count:27460707  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
+          volume id:88  size:1073767976  collection:"collection1"  file_count:14995  delete_count:206  deleted_byte_count:81222360  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615629897 
+          volume id:202  size:1077533160  collection:"collection1"  file_count:2847  delete_count:67  deleted_byte_count:65172985  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615588497 
+          volume id:203  size:1027316272  collection:"collection1"  file_count:3040  delete_count:11  deleted_byte_count:3993230  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631728 
+          volume id:205  size:1078485304  collection:"collection1"  file_count:2869  delete_count:43  deleted_byte_count:18290259  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615579314 
+          volume id:206  size:1082045848  collection:"collection1"  file_count:2979  delete_count:225  deleted_byte_count:88220074  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615564274 
+          volume id:209  size:1074083592  collection:"collection1"  file_count:3238  delete_count:4  deleted_byte_count:1494244  replica_placement:100  version:3  modified_at_second:1615419954 
+          volume id:211  size:1080610712  collection:"collection1"  file_count:3247  delete_count:7  deleted_byte_count:1891456  replica_placement:100  version:3  modified_at_second:1615269124 
+          volume id:212  size:1078293360  collection:"collection1"  file_count:3106  delete_count:6  deleted_byte_count:2085755  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:213  size:1093587976  collection:"collection1"  file_count:3681  delete_count:12  deleted_byte_count:3138791  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:214  size:1074486992  collection:"collection1"  file_count:3217  delete_count:10  deleted_byte_count:6392871  replica_placement:100  version:3  modified_at_second:1615586383 
+          volume id:216  size:1080073496  collection:"collection1"  file_count:3316  delete_count:4  deleted_byte_count:179819  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:222  size:1106623104  collection:"collection1"  file_count:3273  delete_count:11  deleted_byte_count:2114627  replica_placement:100  version:3  modified_at_second:1615586243 
+          volume id:223  size:1075233064  collection:"collection1"  file_count:2966  delete_count:9  deleted_byte_count:744001  replica_placement:100  version:3  modified_at_second:1615586244 
+          volume id:227  size:1106699896  collection:"collection1"  file_count:2827  delete_count:20  deleted_byte_count:5496790  replica_placement:100  version:3  modified_at_second:1615609989 
+          volume id:229  size:1109855312  collection:"collection1"  file_count:2857  delete_count:22  deleted_byte_count:2839883  replica_placement:100  version:3  modified_at_second:1615609988 
+          volume id:230  size:1080722984  collection:"collection1"  file_count:2898  delete_count:15  deleted_byte_count:3929261  replica_placement:100  version:3  modified_at_second:1615610537 
+          volume id:231  size:1112917696  collection:"collection1"  file_count:3151  delete_count:20  deleted_byte_count:2989828  replica_placement:100  version:3  modified_at_second:1615611350 
+          volume id:233  size:1080526464  collection:"collection1"  file_count:3136  delete_count:61  deleted_byte_count:17991717  replica_placement:100  version:3  modified_at_second:1615611352 
+          volume id:234  size:1073835280  collection:"collection1"  file_count:2965  delete_count:41  deleted_byte_count:4960354  replica_placement:100  version:3  modified_at_second:1615611351 
+          volume id:235  size:1075586104  collection:"collection1"  file_count:2767  delete_count:33  deleted_byte_count:3216540  replica_placement:100  version:3  modified_at_second:1615611354 
+          volume id:237  size:375722792  collection:"collection1"  file_count:736  delete_count:16  deleted_byte_count:4464870  replica_placement:100  version:3  modified_at_second:1615631727 
+          volume id:239  size:426569024  collection:"collection1"  file_count:693  delete_count:19  deleted_byte_count:13020783  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630838 
+          volume id:241  size:380217424  collection:"collection1"  file_count:633  delete_count:6  deleted_byte_count:1715768  replica_placement:100  version:3  modified_at_second:1615632006 
+          volume id:244  size:1080295352  collection:"collection2"  file_count:10812  delete_count:1  deleted_byte_count:795  replica_placement:100  version:3  modified_at_second:1614852171 
+          volume id:245  size:1074597056  collection:"collection2"  file_count:10371  delete_count:3  deleted_byte_count:209701  replica_placement:100  version:3  modified_at_second:1614852093 
+          volume id:246  size:1075998648  collection:"collection2"  file_count:10365  delete_count:1  deleted_byte_count:13112  replica_placement:100  version:3  modified_at_second:1614852105 
+          volume id:248  size:1084301184  collection:"collection2"  file_count:11217  delete_count:4  deleted_byte_count:746488  replica_placement:100  version:3  modified_at_second:1614856285 
+          volume id:249  size:1074819136  collection:"collection2"  file_count:10763  delete_count:2  deleted_byte_count:271699  replica_placement:100  version:3  modified_at_second:1614856230 
+          volume id:251  size:1075684488  collection:"collection2"  file_count:10847  replica_placement:100  version:3  modified_at_second:1614856270 
+          volume id:252  size:1075065208  collection:"collection2"  file_count:14622  delete_count:2  deleted_byte_count:5228  replica_placement:100  version:3  modified_at_second:1614861196 
+          volume id:253  size:1087328816  collection:"collection2"  file_count:14920  delete_count:3  deleted_byte_count:522994  replica_placement:100  version:3  modified_at_second:1614861255 
+          volume id:255  size:1079581640  collection:"collection2"  file_count:14877  delete_count:3  deleted_byte_count:101223  replica_placement:100  version:3  modified_at_second:1614861233 
+          volume id:256  size:1074283592  collection:"collection2"  file_count:14157  delete_count:1  deleted_byte_count:18156  replica_placement:100  version:3  modified_at_second:1614861100 
+          volume id:258  size:1075527216  collection:"collection2"  file_count:18421  delete_count:4  deleted_byte_count:267833  replica_placement:100  version:3  modified_at_second:1614866420 
+          volume id:259  size:1075507776  collection:"collection2"  file_count:18079  delete_count:2  deleted_byte_count:71992  replica_placement:100  version:3  modified_at_second:1614866381 
+          volume id:264  size:1081624192  collection:"collection2"  file_count:21151  replica_placement:100  version:3  modified_at_second:1614871629 
+          volume id:265  size:1076401104  collection:"collection2"  file_count:19932  delete_count:2  deleted_byte_count:160823  replica_placement:100  version:3  modified_at_second:1615629130 
+          volume id:266  size:1075617464  collection:"collection2"  file_count:20075  delete_count:1  deleted_byte_count:1039  replica_placement:100  version:3  modified_at_second:1614871526 
+          volume id:267  size:1075699544  collection:"collection2"  file_count:21039  delete_count:3  deleted_byte_count:59956  replica_placement:100  version:3  modified_at_second:1614877294 
+          volume id:268  size:1074490592  collection:"collection2"  file_count:21698  delete_count:1  deleted_byte_count:33968  replica_placement:100  version:3  modified_at_second:1614877434 
+          volume id:269  size:1077552872  collection:"collection2"  file_count:21875  delete_count:4  deleted_byte_count:347272  replica_placement:100  version:3  modified_at_second:1614877481 
+          volume id:270  size:1076876568  collection:"collection2"  file_count:22057  delete_count:1  deleted_byte_count:43916  replica_placement:100  version:3  modified_at_second:1614877469 
+          volume id:275  size:1078349024  collection:"collection2"  file_count:20808  delete_count:1  deleted_byte_count:1118  replica_placement:100  version:3  modified_at_second:1614884147 
+          volume id:277  size:1074956288  collection:"collection2"  file_count:19260  delete_count:2  deleted_byte_count:172356  replica_placement:100  version:3  modified_at_second:1614889988 
+          volume id:278  size:1078798640  collection:"collection2"  file_count:20597  delete_count:5  deleted_byte_count:400060  replica_placement:100  version:3  modified_at_second:1614890292 
+          volume id:279  size:1077325040  collection:"collection2"  file_count:19671  delete_count:6  deleted_byte_count:379116  replica_placement:100  version:3  modified_at_second:1614890229 
+          volume id:280  size:1077432216  collection:"collection2"  file_count:20286  delete_count:1  deleted_byte_count:879  replica_placement:100  version:3  modified_at_second:1614890262 
+          volume id:281  size:1077581096  collection:"collection2"  file_count:20206  delete_count:3  deleted_byte_count:143964  replica_placement:100  version:3  modified_at_second:1614890237 
+          volume id:284  size:1074533384  collection:"collection2"  file_count:22196  delete_count:4  deleted_byte_count:154683  replica_placement:100  version:3  modified_at_second:1614897231 
+          volume id:285  size:1082128688  collection:"collection2"  file_count:21804  delete_count:1  deleted_byte_count:1064  replica_placement:100  version:3  modified_at_second:1614897165 
+          volume id:289  size:1075284256  collection:"collection2"  file_count:29342  delete_count:5  deleted_byte_count:100454  replica_placement:100  version:3  modified_at_second:1614904977 
+          volume id:290  size:1074723792  collection:"collection2"  file_count:28340  delete_count:4  deleted_byte_count:199064  replica_placement:100  version:3  modified_at_second:1614904924 
+          volume id:291  size:1076256768  collection:"collection2"  file_count:28301  delete_count:5  deleted_byte_count:116027  replica_placement:100  version:3  modified_at_second:1614904924 
+          volume id:293  size:1075409792  collection:"collection2"  file_count:26063  delete_count:4  deleted_byte_count:183834  replica_placement:100  version:3  modified_at_second:1614912235 
+          volume id:294  size:1075444048  collection:"collection2"  file_count:26076  delete_count:4  deleted_byte_count:194914  replica_placement:100  version:3  modified_at_second:1614912220 
+          volume id:296  size:1077824032  collection:"collection2"  file_count:26741  delete_count:4  deleted_byte_count:199906  replica_placement:100  version:3  modified_at_second:1614912301 
+          volume id:297  size:1080229136  collection:"collection2"  file_count:23409  delete_count:5  deleted_byte_count:46268  replica_placement:100  version:3  modified_at_second:1614918481 
+          volume id:298  size:1075410136  collection:"collection2"  file_count:23222  delete_count:2  deleted_byte_count:46110  replica_placement:100  version:3  modified_at_second:1614918474 
+          volume id:299  size:1075147936  collection:"collection2"  file_count:22927  delete_count:4  deleted_byte_count:345569  replica_placement:100  version:3  modified_at_second:1614918455 
+          volume id:300  size:1076212392  collection:"collection2"  file_count:22892  delete_count:2  deleted_byte_count:61320  replica_placement:100  version:3  modified_at_second:1614918464 
+          volume id:301  size:1074655600  collection:"collection2"  file_count:22543  delete_count:6  deleted_byte_count:136968  replica_placement:100  version:3  modified_at_second:1614918378 
+          volume id:303  size:1075944480  collection:"collection2"  file_count:22541  delete_count:2  deleted_byte_count:13617  replica_placement:100  version:3  modified_at_second:1614925431 
+          volume id:306  size:1074764016  collection:"collection2"  file_count:22939  replica_placement:100  version:3  modified_at_second:1614925462 
+          volume id:307  size:1076568000  collection:"collection2"  file_count:23377  delete_count:2  deleted_byte_count:25453  replica_placement:100  version:3  modified_at_second:1614931448 
+          volume id:308  size:1074022392  collection:"collection2"  file_count:23086  delete_count:2  deleted_byte_count:2127  replica_placement:100  version:3  modified_at_second:1614931401 
+          volume id:309  size:1075178664  collection:"collection2"  file_count:22692  delete_count:3  deleted_byte_count:171124  replica_placement:100  version:3  modified_at_second:1614931409 
+          volume id:310  size:1074761528  collection:"collection2"  file_count:21441  delete_count:3  deleted_byte_count:13934  replica_placement:100  version:3  modified_at_second:1614931077 
+          volume id:314  size:1074670840  collection:"collection2"  file_count:20964  delete_count:4  deleted_byte_count:304291  replica_placement:100  version:3  modified_at_second:1614937441 
+          volume id:315  size:1084153544  collection:"collection2"  file_count:23638  delete_count:2  deleted_byte_count:53956  replica_placement:100  version:3  modified_at_second:1614937885 
+          volume id:317  size:1076215096  collection:"collection2"  file_count:23572  delete_count:2  deleted_byte_count:1441356  replica_placement:100  version:3  modified_at_second:1614943965 
+          volume id:318  size:1075965168  collection:"collection2"  file_count:22459  delete_count:2  deleted_byte_count:37778  replica_placement:100  version:3  modified_at_second:1614943862 
+          volume id:319  size:1073952880  collection:"collection2"  file_count:22286  delete_count:2  deleted_byte_count:43421  replica_placement:100  version:3  modified_at_second:1614943810 
+          volume id:320  size:1082437792  collection:"collection2"  file_count:21544  delete_count:3  deleted_byte_count:16712  replica_placement:100  version:3  modified_at_second:1614943599 
+          volume id:321  size:1081477904  collection:"collection2"  file_count:23531  delete_count:5  deleted_byte_count:262564  replica_placement:100  version:3  modified_at_second:1614943982 
+          volume id:324  size:1075606680  collection:"collection2"  file_count:20799  delete_count:1  deleted_byte_count:251210  replica_placement:100  version:3  modified_at_second:1614950310 
+          volume id:325  size:1080701144  collection:"collection2"  file_count:21735  replica_placement:100  version:3  modified_at_second:1614950525 
+          volume id:330  size:1080825832  collection:"collection2"  file_count:22464  delete_count:2  deleted_byte_count:15771  replica_placement:100  version:3  modified_at_second:1614956477 
+          volume id:332  size:1075569928  collection:"collection2"  file_count:22097  delete_count:3  deleted_byte_count:98273  replica_placement:100  version:3  modified_at_second:1614962869 
+          volume id:334  size:1075607880  collection:"collection2"  file_count:22546  delete_count:6  deleted_byte_count:101538  replica_placement:100  version:3  modified_at_second:1614962978 
+          volume id:336  size:1087853056  collection:"collection2"  file_count:22801  delete_count:2  deleted_byte_count:26394  replica_placement:100  version:3  modified_at_second:1614963005 
+          volume id:337  size:1075646784  collection:"collection2"  file_count:21934  delete_count:1  deleted_byte_count:3397  replica_placement:100  version:3  modified_at_second:1614969937 
+          volume id:338  size:1076118304  collection:"collection2"  file_count:21680  replica_placement:100  version:3  modified_at_second:1614969850 
+          volume id:340  size:1079462184  collection:"collection2"  file_count:22319  delete_count:4  deleted_byte_count:93620  replica_placement:100  version:3  modified_at_second:1614969978 
+          volume id:341  size:1074448400  collection:"collection2"  file_count:21590  delete_count:5  deleted_byte_count:160085  replica_placement:100  version:3  modified_at_second:1614969858 
+          volume id:342  size:1080186424  collection:"collection2"  file_count:22405  delete_count:4  deleted_byte_count:64819  replica_placement:100  version:3  modified_at_second:1614977521 
+          volume id:344  size:1075035416  collection:"collection2"  file_count:21765  delete_count:1  deleted_byte_count:24623  replica_placement:100  version:3  modified_at_second:1614977465 
+          volume id:345  size:1074560760  collection:"collection2"  file_count:22117  delete_count:2  deleted_byte_count:373286  replica_placement:100  version:3  modified_at_second:1614977457 
+          volume id:346  size:1076464112  collection:"collection2"  file_count:22320  delete_count:4  deleted_byte_count:798258  replica_placement:100  version:3  modified_at_second:1615631322 
+          volume id:348  size:1080623640  collection:"collection2"  file_count:21667  delete_count:1  deleted_byte_count:2443  replica_placement:100  version:3  modified_at_second:1614984606 
+          volume id:350  size:1074756688  collection:"collection2"  file_count:21990  delete_count:3  deleted_byte_count:233881  replica_placement:100  version:3  modified_at_second:1614984682 
+          volume id:351  size:1078795112  collection:"collection2"  file_count:23660  delete_count:3  deleted_byte_count:102141  replica_placement:100  version:3  modified_at_second:1614984816 
+          volume id:352  size:1077145936  collection:"collection2"  file_count:22066  delete_count:1  deleted_byte_count:1018  replica_placement:100  version:3  modified_at_second:1614992130 
+          volume id:353  size:1074897496  collection:"collection2"  file_count:21266  delete_count:2  deleted_byte_count:3105374  replica_placement:100  version:3  modified_at_second:1614991951 
+          volume id:355  size:1075239728  collection:"collection2"  file_count:22244  delete_count:1  deleted_byte_count:23282  replica_placement:100  version:3  modified_at_second:1614992157 
+          volume id:356  size:1083305048  collection:"collection2"  file_count:21552  delete_count:4  deleted_byte_count:14472  replica_placement:100  version:3  modified_at_second:1614992028 
+          volume id:358  size:1085152368  collection:"collection2"  file_count:23756  delete_count:3  deleted_byte_count:44531  replica_placement:100  version:3  modified_at_second:1614998824 
+          volume id:360  size:1075532456  collection:"collection2"  file_count:22574  delete_count:3  deleted_byte_count:1774776  replica_placement:100  version:3  modified_at_second:1614998770 
+          volume id:361  size:1075362744  collection:"collection2"  file_count:22272  delete_count:1  deleted_byte_count:3497  replica_placement:100  version:3  modified_at_second:1614998668 
+          volume id:375  size:1076140568  collection:"collection2"  file_count:21880  delete_count:2  deleted_byte_count:51103  replica_placement:100  version:3  modified_at_second:1615016787 
+          volume id:376  size:1074845944  collection:"collection2"  file_count:22908  delete_count:4  deleted_byte_count:432305  replica_placement:100  version:3  modified_at_second:1615016916 
+          volume id:377  size:957284144  collection:"collection2"  file_count:14923  delete_count:1  deleted_byte_count:1797  replica_placement:100  version:3  modified_at_second:1615632323 
+          volume id:378  size:959273936  collection:"collection2"  file_count:15027  delete_count:4  deleted_byte_count:231414  replica_placement:100  version:3  modified_at_second:1615632323 
+          volume id:381  size:939261032  collection:"collection2"  file_count:14615  delete_count:5  deleted_byte_count:1192272  replica_placement:100  version:3  modified_at_second:1615632324 
+        Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
+      DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
+    Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
+  DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 
+  DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0)
+    Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0)
+      DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0)
+        Disk hdd(volume:267/2000 active:267 free:1733 remote:0)
+          volume id:1  size:284693256  file_count:1558  delete_count:2  deleted_byte_count:4818  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632062 
+          volume id:2  size:289228560  file_count:1640  delete_count:4  deleted_byte_count:464508  replica_placement:100  version:3  compact_revision:6  modified_at_second:1615630622 
+          volume id:3  size:308741952  file_count:1637  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632242 
+          volume id:4  size:285986968  file_count:1640  delete_count:1  deleted_byte_count:145095  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632302 
+          volume id:5  size:293806008  file_count:1669  delete_count:2  deleted_byte_count:274334  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631342 
+          volume id:6  size:302411024  file_count:1604  delete_count:2  deleted_byte_count:274587  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631402 
+          volume id:7  size:1924728  collection:"collection4"  file_count:15  replica_placement:100  version:3  modified_at_second:1609331040 
+          volume id:9  size:77337416  collection:"collection3"  file_count:58  replica_placement:100  version:3  ttl:772  modified_at_second:1615513762 
+          volume id:10  size:1212784656  collection:"collection0"  file_count:58  replica_placement:100  version:3  modified_at_second:1609814543 
+          volume id:11  size:1109224552  collection:"collection0"  file_count:44  replica_placement:100  version:3  modified_at_second:1609815123 
+          volume id:12  size:1110923848  collection:"collection0"  file_count:45  replica_placement:100  version:3  modified_at_second:1609819726 
+          volume id:13  size:1184910656  collection:"collection0"  file_count:47  replica_placement:100  version:3  modified_at_second:1609827832 
+          volume id:14  size:1107475720  collection:"collection0"  file_count:80  delete_count:3  deleted_byte_count:6870  replica_placement:100  version:3  modified_at_second:1612956983 
+          volume id:15  size:1115965160  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:4956  replica_placement:100  version:3  modified_at_second:1612957001 
+          volume id:16  size:1113666048  collection:"collection0"  file_count:73  delete_count:5  deleted_byte_count:6318  replica_placement:100  version:3  modified_at_second:1612957007 
+          volume id:17  size:1095115800  collection:"collection0"  file_count:83  delete_count:3  deleted_byte_count:7099  replica_placement:100  version:3  modified_at_second:1612957000 
+          volume id:18  size:1096678688  collection:"collection0"  file_count:88  delete_count:4  deleted_byte_count:8633  replica_placement:100  version:3  modified_at_second:1612957000 
+          volume id:19  size:1096923792  collection:"collection0"  file_count:100  delete_count:10  deleted_byte_count:75779917  replica_placement:100  version:3  compact_revision:4  modified_at_second:1612957011 
+          volume id:20  size:1074760432  collection:"collection0"  file_count:82  delete_count:5  deleted_byte_count:12156  replica_placement:100  version:3  compact_revision:2  modified_at_second:1612957011 
+          volume id:22  size:1086828368  collection:"collection0"  file_count:75  delete_count:3  deleted_byte_count:5551  replica_placement:100  version:3  modified_at_second:1612957007 
+          volume id:23  size:1076380280  collection:"collection0"  file_count:68  delete_count:2  deleted_byte_count:2910  replica_placement:100  version:3  modified_at_second:1612957011 
+          volume id:24  size:1074139808  collection:"collection0"  file_count:90  delete_count:1  deleted_byte_count:1977  replica_placement:100  version:3  modified_at_second:1612957011 
+          volume id:25  size:690757544  collection:"collection0"  file_count:38  delete_count:1  deleted_byte_count:1944  replica_placement:100  version:3  modified_at_second:1612956995 
+          volume id:26  size:532657632  collection:"collection0"  file_count:100  delete_count:4  deleted_byte_count:9081  replica_placement:100  version:3  modified_at_second:1614170023 
+          volume id:34  size:1077111136  collection:"collection1"  file_count:9781  delete_count:110  deleted_byte_count:20894827  replica_placement:100  version:3  modified_at_second:1615619366 
+          volume id:35  size:1075241656  collection:"collection1"  file_count:10523  delete_count:96  deleted_byte_count:46618989  replica_placement:100  version:3  modified_at_second:1615618790 
+          volume id:36  size:1075118360  collection:"collection1"  file_count:10342  delete_count:116  deleted_byte_count:25493106  replica_placement:100  version:3  modified_at_second:1615606148 
+          volume id:37  size:1075895584  collection:"collection1"  file_count:12013  delete_count:98  deleted_byte_count:50747932  replica_placement:100  version:3  modified_at_second:1615594777 
+          volume id:39  size:1076606536  collection:"collection1"  file_count:12612  delete_count:78  deleted_byte_count:17462730  replica_placement:100  version:3  modified_at_second:1615611959 
+          volume id:40  size:1075358552  collection:"collection1"  file_count:12597  delete_count:62  deleted_byte_count:11657901  replica_placement:100  version:3  modified_at_second:1615612994 
+          volume id:41  size:1076283528  collection:"collection1"  file_count:12088  delete_count:84  deleted_byte_count:19319268  replica_placement:100  version:3  modified_at_second:1615596736 
+          volume id:42  size:1093948352  collection:"collection1"  file_count:7889  delete_count:47  deleted_byte_count:5697275  replica_placement:100  version:3  modified_at_second:1615548908 
+          volume id:43  size:1116445864  collection:"collection1"  file_count:7358  delete_count:54  deleted_byte_count:9534379  replica_placement:100  version:3  modified_at_second:1615566170 
+          volume id:44  size:1077582560  collection:"collection1"  file_count:7295  delete_count:50  deleted_byte_count:12618414  replica_placement:100  version:3  modified_at_second:1615566170 
+          volume id:45  size:1075254640  collection:"collection1"  file_count:10772  delete_count:76  deleted_byte_count:22426345  replica_placement:100  version:3  modified_at_second:1615573499 
+          volume id:46  size:1075286056  collection:"collection1"  file_count:9947  delete_count:309  deleted_byte_count:105601163  replica_placement:100  version:3  modified_at_second:1615569826 
+          volume id:48  size:1076778720  collection:"collection1"  file_count:9850  delete_count:77  deleted_byte_count:16641907  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630690 
+          volume id:50  size:1076688224  collection:"collection1"  file_count:7921  delete_count:26  deleted_byte_count:5162032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615610879 
+          volume id:52  size:1083529704  collection:"collection1"  file_count:10128  delete_count:32  deleted_byte_count:10608391  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615599195 
+          volume id:53  size:1063089216  collection:"collection1"  file_count:9832  delete_count:31  deleted_byte_count:9273066  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
+          volume id:55  size:1012890016  collection:"collection1"  file_count:8651  delete_count:27  deleted_byte_count:9418841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631452 
+          volume id:57  size:839849792  collection:"collection1"  file_count:7514  delete_count:24  deleted_byte_count:6228543  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631774 
+          volume id:58  size:908064200  collection:"collection1"  file_count:8128  delete_count:21  deleted_byte_count:6113731  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632342 
+          volume id:59  size:988302272  collection:"collection1"  file_count:8098  delete_count:20  deleted_byte_count:3947615  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632238 
+          volume id:60  size:1010702480  collection:"collection1"  file_count:8969  delete_count:79  deleted_byte_count:24782814  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632439 
+          volume id:61  size:975604488  collection:"collection1"  file_count:8683  delete_count:20  deleted_byte_count:10276072  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631176 
+          volume id:62  size:873845936  collection:"collection1"  file_count:7897  delete_count:23  deleted_byte_count:10920170  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631133 
+          volume id:64  size:965638488  collection:"collection1"  file_count:8218  delete_count:27  deleted_byte_count:6922489  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631031 
+          volume id:65  size:823283552  collection:"collection1"  file_count:7834  delete_count:29  deleted_byte_count:5950610  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632306 
+          volume id:66  size:821343440  collection:"collection1"  file_count:7383  delete_count:29  deleted_byte_count:12010343  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631968 
+          volume id:67  size:878713872  collection:"collection1"  file_count:7299  delete_count:117  deleted_byte_count:24857326  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
+          volume id:68  size:898630584  collection:"collection1"  file_count:6934  delete_count:95  deleted_byte_count:27460707  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
+          volume id:70  size:886695472  collection:"collection1"  file_count:7769  delete_count:164  deleted_byte_count:45162513  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632398 
+          volume id:71  size:907608392  collection:"collection1"  file_count:7658  delete_count:122  deleted_byte_count:27622941  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
+          volume id:72  size:903990720  collection:"collection1"  file_count:6996  delete_count:240  deleted_byte_count:74147727  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630982 
+          volume id:73  size:929047664  collection:"collection1"  file_count:7038  delete_count:227  deleted_byte_count:65336664  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630707 
+          volume id:74  size:957046128  collection:"collection1"  file_count:6981  delete_count:259  deleted_byte_count:73080838  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631460 
+          volume id:75  size:908044992  collection:"collection1"  file_count:6911  delete_count:268  deleted_byte_count:73934373  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632430 
+          volume id:76  size:985296744  collection:"collection1"  file_count:6566  delete_count:61  deleted_byte_count:44464430  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
+          volume id:77  size:929398296  collection:"collection1"  file_count:7427  delete_count:238  deleted_byte_count:59581579  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632013 
+          volume id:78  size:1075671512  collection:"collection1"  file_count:7540  delete_count:258  deleted_byte_count:71726846  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615582829 
+          volume id:79  size:948225472  collection:"collection1"  file_count:6997  delete_count:227  deleted_byte_count:60625763  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631326 
+          volume id:82  size:1041661800  collection:"collection1"  file_count:7043  delete_count:207  deleted_byte_count:52275724  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632430 
+          volume id:83  size:936195856  collection:"collection1"  file_count:7593  delete_count:13  deleted_byte_count:4633917  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632029 
+          volume id:85  size:1023867520  collection:"collection1"  file_count:7787  delete_count:240  deleted_byte_count:82091742  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631723 
+          volume id:86  size:1009437488  collection:"collection1"  file_count:8474  delete_count:236  deleted_byte_count:64543674  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
+          volume id:87  size:922276640  collection:"collection1"  file_count:12902  delete_count:13  deleted_byte_count:3412959  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632438 
+          volume id:89  size:1044401976  collection:"collection1"  file_count:14943  delete_count:243  deleted_byte_count:58543159  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632208 
+          volume id:90  size:891145784  collection:"collection1"  file_count:14608  delete_count:10  deleted_byte_count:2564369  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615629390 
+          volume id:91  size:936572832  collection:"collection1"  file_count:14686  delete_count:11  deleted_byte_count:4717727  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631851 
+          volume id:92  size:992440712  collection:"collection1"  file_count:7061  delete_count:195  deleted_byte_count:60649573  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630566 
+          volume id:93  size:1079603768  collection:"collection1"  file_count:7878  delete_count:270  deleted_byte_count:74150048  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615556015 
+          volume id:94  size:1030685824  collection:"collection1"  file_count:7660  delete_count:207  deleted_byte_count:70150733  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631616 
+          volume id:95  size:990879168  collection:"collection1"  file_count:6620  delete_count:206  deleted_byte_count:60363604  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631866 
+          volume id:96  size:989296136  collection:"collection1"  file_count:7544  delete_count:229  deleted_byte_count:59931853  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630778 
+          volume id:97  size:1053112992  collection:"collection1"  file_count:6789  delete_count:50  deleted_byte_count:38894001  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631194 
+          volume id:99  size:1071718504  collection:"collection1"  file_count:7470  delete_count:8  deleted_byte_count:9624950  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631175 
+          volume id:100  size:1083617440  collection:"collection1"  file_count:7018  delete_count:187  deleted_byte_count:61304236  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615505917 
+          volume id:101  size:1077109520  collection:"collection1"  file_count:7706  delete_count:226  deleted_byte_count:77864841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630994 
+          volume id:102  size:1074359920  collection:"collection1"  file_count:7338  delete_count:7  deleted_byte_count:6499151  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615626683 
+          volume id:103  size:1075863904  collection:"collection1"  file_count:7184  delete_count:186  deleted_byte_count:58872238  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615628417 
+          volume id:104  size:1076383768  collection:"collection1"  file_count:7663  delete_count:184  deleted_byte_count:100578087  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615602661 
+          volume id:105  size:1073996824  collection:"collection1"  file_count:6873  delete_count:19  deleted_byte_count:14271533  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615499756 
+          volume id:108  size:1074648024  collection:"collection1"  file_count:7472  delete_count:194  deleted_byte_count:70864699  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593232 
+          volume id:109  size:1075254560  collection:"collection1"  file_count:7556  delete_count:263  deleted_byte_count:55155265  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615502487 
+          volume id:110  size:1076575744  collection:"collection1"  file_count:6996  delete_count:163  deleted_byte_count:52954032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615590786 
+          volume id:111  size:1073826232  collection:"collection1"  file_count:7355  delete_count:155  deleted_byte_count:50083578  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593233 
+          volume id:114  size:1074762784  collection:"collection1"  file_count:8802  delete_count:156  deleted_byte_count:38470055  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615591826 
+          volume id:115  size:1076192240  collection:"collection1"  file_count:7690  delete_count:154  deleted_byte_count:32267193  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615285295 
+          volume id:116  size:1074489504  collection:"collection1"  file_count:9981  delete_count:174  deleted_byte_count:53998777  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615611567 
+          volume id:117  size:1073917192  collection:"collection1"  file_count:9520  delete_count:114  deleted_byte_count:21835126  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573714 
+          volume id:118  size:1074064400  collection:"collection1"  file_count:8738  delete_count:15  deleted_byte_count:3460697  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615516265 
+          volume id:119  size:1075940104  collection:"collection1"  file_count:9003  delete_count:12  deleted_byte_count:9128155  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573880 
+          volume id:120  size:1076115928  collection:"collection1"  file_count:9639  delete_count:118  deleted_byte_count:33357871  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615482567 
+          volume id:121  size:1078803248  collection:"collection1"  file_count:10113  delete_count:441  deleted_byte_count:94128627  replica_placement:100  version:3  modified_at_second:1615506629 
+          volume id:122  size:1076235312  collection:"collection1"  file_count:9106  delete_count:252  deleted_byte_count:93041272  replica_placement:100  version:3  modified_at_second:1615585913 
+          volume id:123  size:1080491112  collection:"collection1"  file_count:10623  delete_count:302  deleted_byte_count:83956998  replica_placement:100  version:3  modified_at_second:1615585916 
+          volume id:124  size:1074519360  collection:"collection1"  file_count:9457  delete_count:286  deleted_byte_count:74752459  replica_placement:100  version:3  modified_at_second:1615585916 
+          volume id:125  size:1088687040  collection:"collection1"  file_count:9518  delete_count:281  deleted_byte_count:76037905  replica_placement:100  version:3  modified_at_second:1615585913 
+          volume id:126  size:1073867464  collection:"collection1"  file_count:9320  delete_count:278  deleted_byte_count:94547424  replica_placement:100  version:3  modified_at_second:1615585912 
+          volume id:127  size:1074907336  collection:"collection1"  file_count:9900  delete_count:133  deleted_byte_count:48570820  replica_placement:100  version:3  modified_at_second:1615612991 
+          volume id:129  size:1074704272  collection:"collection1"  file_count:10012  delete_count:150  deleted_byte_count:64491721  replica_placement:100  version:3  modified_at_second:1615627566 
+          volume id:130  size:1075000632  collection:"collection1"  file_count:10633  delete_count:161  deleted_byte_count:34768201  replica_placement:100  version:3  modified_at_second:1615582330 
+          volume id:131  size:1075279584  collection:"collection1"  file_count:10075  delete_count:135  deleted_byte_count:29795712  replica_placement:100  version:3  modified_at_second:1615523898 
+          volume id:132  size:1088539496  collection:"collection1"  file_count:11051  delete_count:71  deleted_byte_count:17178322  replica_placement:100  version:3  modified_at_second:1615619584 
+          volume id:133  size:1075952760  collection:"collection1"  file_count:9538  delete_count:74  deleted_byte_count:19558008  replica_placement:100  version:3  modified_at_second:1615584780 
+          volume id:134  size:1074367304  collection:"collection1"  file_count:10662  delete_count:69  deleted_byte_count:25530139  replica_placement:100  version:3  modified_at_second:1615555876 
+          volume id:135  size:1073906720  collection:"collection1"  file_count:10446  delete_count:71  deleted_byte_count:28599975  replica_placement:100  version:3  modified_at_second:1615569816 
+          volume id:137  size:1074309264  collection:"collection1"  file_count:9633  delete_count:50  deleted_byte_count:27487972  replica_placement:100  version:3  modified_at_second:1615572231 
+          volume id:139  size:1074163936  collection:"collection1"  file_count:9314  delete_count:43  deleted_byte_count:10631353  replica_placement:100  version:3  modified_at_second:1615571946 
+          volume id:141  size:1074619488  collection:"collection1"  file_count:9840  delete_count:45  deleted_byte_count:40890181  replica_placement:100  version:3  modified_at_second:1615630994 
+          volume id:142  size:1075732992  collection:"collection1"  file_count:9009  delete_count:48  deleted_byte_count:9912854  replica_placement:100  version:3  modified_at_second:1615598914 
+          volume id:143  size:1075011280  collection:"collection1"  file_count:9608  delete_count:51  deleted_byte_count:37282460  replica_placement:100  version:3  modified_at_second:1615488586 
+          volume id:145  size:1074394928  collection:"collection1"  file_count:9255  delete_count:34  deleted_byte_count:38011392  replica_placement:100  version:3  modified_at_second:1615591825 
+          volume id:146  size:1076337520  collection:"collection1"  file_count:10492  delete_count:50  deleted_byte_count:17071505  replica_placement:100  version:3  modified_at_second:1615632005 
+          volume id:147  size:1077130544  collection:"collection1"  file_count:10451  delete_count:27  deleted_byte_count:8290907  replica_placement:100  version:3  modified_at_second:1615604117 
+          volume id:148  size:1076066568  collection:"collection1"  file_count:9547  delete_count:33  deleted_byte_count:7034089  replica_placement:100  version:3  modified_at_second:1615586393 
+          volume id:149  size:1074989016  collection:"collection1"  file_count:8352  delete_count:35  deleted_byte_count:7179742  replica_placement:100  version:3  modified_at_second:1615494496 
+          volume id:150  size:1076290408  collection:"collection1"  file_count:9328  delete_count:33  deleted_byte_count:43417791  replica_placement:100  version:3  modified_at_second:1615611569 
+          volume id:151  size:1098659752  collection:"collection1"  file_count:10805  delete_count:27  deleted_byte_count:7209106  replica_placement:100  version:3  modified_at_second:1615586390 
+          volume id:152  size:1075941376  collection:"collection1"  file_count:9951  delete_count:36  deleted_byte_count:25348335  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:153  size:1078539784  collection:"collection1"  file_count:10924  delete_count:34  deleted_byte_count:12603081  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:154  size:1081244752  collection:"collection1"  file_count:11002  delete_count:31  deleted_byte_count:8467560  replica_placement:100  version:3  modified_at_second:1615478471 
+          volume id:156  size:1074975832  collection:"collection1"  file_count:9535  delete_count:40  deleted_byte_count:11426621  replica_placement:100  version:3  modified_at_second:1615628342 
+          volume id:157  size:1076758536  collection:"collection1"  file_count:10012  delete_count:19  deleted_byte_count:11688737  replica_placement:100  version:3  modified_at_second:1615597782 
+          volume id:158  size:1087251976  collection:"collection1"  file_count:9972  delete_count:20  deleted_byte_count:10328429  replica_placement:100  version:3  modified_at_second:1615588879 
+          volume id:159  size:1074132336  collection:"collection1"  file_count:9382  delete_count:27  deleted_byte_count:11474574  replica_placement:100  version:3  modified_at_second:1615593593 
+          volume id:160  size:1075680976  collection:"collection1"  file_count:9772  delete_count:22  deleted_byte_count:4981968  replica_placement:100  version:3  modified_at_second:1615597782 
+          volume id:161  size:1077397136  collection:"collection1"  file_count:9988  delete_count:28  deleted_byte_count:12509164  replica_placement:100  version:3  modified_at_second:1615631452 
+          volume id:162  size:1074286880  collection:"collection1"  file_count:11220  delete_count:17  deleted_byte_count:1815547  replica_placement:100  version:3  modified_at_second:1615478127 
+          volume id:163  size:1074457224  collection:"collection1"  file_count:12524  delete_count:27  deleted_byte_count:6359619  replica_placement:100  version:3  modified_at_second:1615579313 
+          volume id:164  size:1074261256  collection:"collection1"  file_count:11922  delete_count:25  deleted_byte_count:2923288  replica_placement:100  version:3  modified_at_second:1615620085 
+          volume id:165  size:1073891080  collection:"collection1"  file_count:9152  delete_count:12  deleted_byte_count:19164659  replica_placement:100  version:3  modified_at_second:1615471910 
+          volume id:166  size:1075637536  collection:"collection1"  file_count:14211  delete_count:24  deleted_byte_count:20415490  replica_placement:100  version:3  modified_at_second:1615491021 
+          volume id:167  size:1073958280  collection:"collection1"  file_count:25231  delete_count:48  deleted_byte_count:26022344  replica_placement:100  version:3  modified_at_second:1615620014 
+          volume id:168  size:1074718864  collection:"collection1"  file_count:25702  delete_count:40  deleted_byte_count:4024775  replica_placement:100  version:3  modified_at_second:1615585664 
+          volume id:169  size:1073863032  collection:"collection1"  file_count:25248  delete_count:43  deleted_byte_count:3013817  replica_placement:100  version:3  modified_at_second:1615569832 
+          volume id:170  size:1075747088  collection:"collection1"  file_count:24596  delete_count:41  deleted_byte_count:3494711  replica_placement:100  version:3  modified_at_second:1615579207 
+          volume id:171  size:1081881400  collection:"collection1"  file_count:24215  delete_count:36  deleted_byte_count:3191335  replica_placement:100  version:3  modified_at_second:1615596486 
+          volume id:172  size:1074787304  collection:"collection1"  file_count:31236  delete_count:50  deleted_byte_count:3316482  replica_placement:100  version:3  modified_at_second:1615612385 
+          volume id:174  size:1073824160  collection:"collection1"  file_count:30689  delete_count:36  deleted_byte_count:2160116  replica_placement:100  version:3  modified_at_second:1615598914 
+          volume id:175  size:1077742472  collection:"collection1"  file_count:32353  delete_count:33  deleted_byte_count:1861403  replica_placement:100  version:3  modified_at_second:1615559516 
+          volume id:176  size:1073854800  collection:"collection1"  file_count:30582  delete_count:34  deleted_byte_count:7701976  replica_placement:100  version:3  modified_at_second:1615626169 
+          volume id:178  size:1087560112  collection:"collection1"  file_count:23482  delete_count:22  deleted_byte_count:18810492  replica_placement:100  version:3  modified_at_second:1615541369 
+          volume id:179  size:1074313920  collection:"collection1"  file_count:21829  delete_count:24  deleted_byte_count:45574435  replica_placement:100  version:3  modified_at_second:1615580308 
+          volume id:180  size:1078438448  collection:"collection1"  file_count:23614  delete_count:12  deleted_byte_count:4496474  replica_placement:100  version:3  modified_at_second:1614773243 
+          volume id:181  size:1074571672  collection:"collection1"  file_count:22898  delete_count:19  deleted_byte_count:6628413  replica_placement:100  version:3  modified_at_second:1614745117 
+          volume id:183  size:1076361616  collection:"collection1"  file_count:31293  delete_count:16  deleted_byte_count:468841  replica_placement:100  version:3  modified_at_second:1615572985 
+          volume id:184  size:1074594216  collection:"collection1"  file_count:31368  delete_count:22  deleted_byte_count:857453  replica_placement:100  version:3  modified_at_second:1615586578 
+          volume id:185  size:1074099592  collection:"collection1"  file_count:30612  delete_count:17  deleted_byte_count:2610847  replica_placement:100  version:3  modified_at_second:1615506835 
+          volume id:186  size:1074220664  collection:"collection1"  file_count:31450  delete_count:15  deleted_byte_count:391855  replica_placement:100  version:3  modified_at_second:1615614934 
+          volume id:187  size:1074396112  collection:"collection1"  file_count:31853  delete_count:17  deleted_byte_count:454283  replica_placement:100  version:3  modified_at_second:1615590491 
+          volume id:188  size:1074732632  collection:"collection1"  file_count:31867  delete_count:19  deleted_byte_count:393743  replica_placement:100  version:3  modified_at_second:1615487645 
+          volume id:189  size:1074847824  collection:"collection1"  file_count:31450  delete_count:16  deleted_byte_count:1040552  replica_placement:100  version:3  modified_at_second:1615335661 
+          volume id:190  size:1074008968  collection:"collection1"  file_count:31987  delete_count:11  deleted_byte_count:685125  replica_placement:100  version:3  modified_at_second:1615447162 
+          volume id:191  size:1075492960  collection:"collection1"  file_count:31301  delete_count:19  deleted_byte_count:708401  replica_placement:100  version:3  modified_at_second:1615357457 
+          volume id:192  size:1075857384  collection:"collection1"  file_count:31490  delete_count:25  deleted_byte_count:720617  replica_placement:100  version:3  modified_at_second:1615621632 
+          volume id:193  size:1076616760  collection:"collection1"  file_count:31907  delete_count:16  deleted_byte_count:464900  replica_placement:100  version:3  modified_at_second:1615507877 
+          volume id:194  size:1073985792  collection:"collection1"  file_count:31434  delete_count:18  deleted_byte_count:391432  replica_placement:100  version:3  modified_at_second:1615559502 
+          volume id:195  size:1074158304  collection:"collection1"  file_count:31453  delete_count:15  deleted_byte_count:718266  replica_placement:100  version:3  modified_at_second:1615559331 
+          volume id:196  size:1074594640  collection:"collection1"  file_count:31665  delete_count:18  deleted_byte_count:3468922  replica_placement:100  version:3  modified_at_second:1615501690 
+          volume id:198  size:1075104624  collection:"collection1"  file_count:16577  delete_count:18  deleted_byte_count:6583181  replica_placement:100  version:3  modified_at_second:1615623371 
+          volume id:199  size:1078117688  collection:"collection1"  file_count:16497  delete_count:14  deleted_byte_count:1514286  replica_placement:100  version:3  modified_at_second:1615585987 
+          volume id:200  size:1075630464  collection:"collection1"  file_count:16380  delete_count:18  deleted_byte_count:1103109  replica_placement:100  version:3  modified_at_second:1615485252 
+          volume id:201  size:1091460440  collection:"collection1"  file_count:16684  delete_count:26  deleted_byte_count:5590335  replica_placement:100  version:3  modified_at_second:1615585987 
+          volume id:204  size:1079766904  collection:"collection1"  file_count:3233  delete_count:255  deleted_byte_count:104707641  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615565702 
+          volume id:207  size:1081939960  collection:"collection1"  file_count:3010  delete_count:4  deleted_byte_count:692350  replica_placement:100  version:3  modified_at_second:1615269061 
+          volume id:208  size:1077863624  collection:"collection1"  file_count:3147  delete_count:6  deleted_byte_count:858726  replica_placement:100  version:3  modified_at_second:1615495515 
+          volume id:209  size:1074083592  collection:"collection1"  file_count:3238  delete_count:4  deleted_byte_count:1494244  replica_placement:100  version:3  modified_at_second:1615419954 
+          volume id:210  size:1094311304  collection:"collection1"  file_count:3468  delete_count:4  deleted_byte_count:466433  replica_placement:100  version:3  modified_at_second:1615495515 
+          volume id:211  size:1080610712  collection:"collection1"  file_count:3247  delete_count:7  deleted_byte_count:1891456  replica_placement:100  version:3  modified_at_second:1615269124 
+          volume id:216  size:1080073496  collection:"collection1"  file_count:3316  delete_count:4  deleted_byte_count:179819  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:218  size:1081263944  collection:"collection1"  file_count:3433  delete_count:14  deleted_byte_count:3454237  replica_placement:100  version:3  modified_at_second:1615603637 
+          volume id:220  size:1081928312  collection:"collection1"  file_count:3166  delete_count:13  deleted_byte_count:4127709  replica_placement:100  version:3  modified_at_second:1615579317 
+          volume id:221  size:1106545536  collection:"collection1"  file_count:3153  delete_count:11  deleted_byte_count:1496835  replica_placement:100  version:3  modified_at_second:1615269138 
+          volume id:224  size:1093691520  collection:"collection1"  file_count:3463  delete_count:10  deleted_byte_count:1128328  replica_placement:100  version:3  modified_at_second:1615601870 
+          volume id:225  size:1080698928  collection:"collection1"  file_count:3115  delete_count:7  deleted_byte_count:18170416  replica_placement:100  version:3  modified_at_second:1615434685 
+          volume id:226  size:1103504792  collection:"collection1"  file_count:2965  delete_count:10  deleted_byte_count:2639254  replica_placement:100  version:3  modified_at_second:1615601870 
+          volume id:227  size:1106699864  collection:"collection1"  file_count:2827  delete_count:19  deleted_byte_count:5393310  replica_placement:100  version:3  modified_at_second:1615609989 
+          volume id:228  size:1109784072  collection:"collection1"  file_count:2504  delete_count:24  deleted_byte_count:5458950  replica_placement:100  version:3  modified_at_second:1615610489 
+          volume id:229  size:1109855256  collection:"collection1"  file_count:2857  delete_count:22  deleted_byte_count:2839883  replica_placement:100  version:3  modified_at_second:1615609989 
+          volume id:231  size:1112917664  collection:"collection1"  file_count:3151  delete_count:19  deleted_byte_count:2852517  replica_placement:100  version:3  modified_at_second:1615611350 
+          volume id:232  size:1073901520  collection:"collection1"  file_count:3004  delete_count:54  deleted_byte_count:10273081  replica_placement:100  version:3  modified_at_second:1615611352 
+          volume id:233  size:1080526464  collection:"collection1"  file_count:3136  delete_count:61  deleted_byte_count:17991717  replica_placement:100  version:3  modified_at_second:1615611354 
+          volume id:236  size:1089476200  collection:"collection1"  file_count:3231  delete_count:53  deleted_byte_count:11625921  replica_placement:100  version:3  modified_at_second:1615611351 
+          volume id:238  size:354320000  collection:"collection1"  file_count:701  delete_count:17  deleted_byte_count:5940420  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632030 
+          volume id:240  size:424791528  collection:"collection1"  file_count:734  delete_count:12  deleted_byte_count:7353071  replica_placement:100  version:3  modified_at_second:1615631669 
+          volume id:242  size:1075383304  collection:"collection2"  file_count:10470  replica_placement:100  version:3  modified_at_second:1614852115 
+          volume id:243  size:1088174560  collection:"collection2"  file_count:11109  delete_count:1  deleted_byte_count:938  replica_placement:100  version:3  modified_at_second:1614852202 
+          volume id:245  size:1074597056  collection:"collection2"  file_count:10371  delete_count:3  deleted_byte_count:209701  replica_placement:100  version:3  modified_at_second:1614852093 
+          volume id:247  size:1075859784  collection:"collection2"  file_count:10443  delete_count:2  deleted_byte_count:564486  replica_placement:100  version:3  modified_at_second:1614856152 
+          volume id:249  size:1074819168  collection:"collection2"  file_count:10763  delete_count:2  deleted_byte_count:271699  replica_placement:100  version:3  modified_at_second:1614856231 
+          volume id:250  size:1080572256  collection:"collection2"  file_count:10220  replica_placement:100  version:3  modified_at_second:1614856129 
+          volume id:251  size:1075684408  collection:"collection2"  file_count:10847  replica_placement:100  version:3  modified_at_second:1614856270 
+          volume id:254  size:1074830800  collection:"collection2"  file_count:14140  delete_count:2  deleted_byte_count:105892  replica_placement:100  version:3  modified_at_second:1614861115 
+          volume id:257  size:1082621664  collection:"collection2"  file_count:18172  delete_count:2  deleted_byte_count:25125  replica_placement:100  version:3  modified_at_second:1614866395 
+          volume id:260  size:1075105664  collection:"collection2"  file_count:17316  delete_count:4  deleted_byte_count:2015310  replica_placement:100  version:3  modified_at_second:1614866226 
+          volume id:261  size:1076628592  collection:"collection2"  file_count:18355  delete_count:1  deleted_byte_count:1155  replica_placement:100  version:3  modified_at_second:1614866420 
+          volume id:262  size:1078492464  collection:"collection2"  file_count:20390  delete_count:3  deleted_byte_count:287601  replica_placement:100  version:3  modified_at_second:1614871601 
+          volume id:263  size:1077167440  collection:"collection2"  file_count:20227  delete_count:4  deleted_byte_count:97887  replica_placement:100  version:3  modified_at_second:1614871567 
+          volume id:268  size:1074490592  collection:"collection2"  file_count:21698  delete_count:1  deleted_byte_count:33968  replica_placement:100  version:3  modified_at_second:1614877435 
+          volume id:269  size:1077552720  collection:"collection2"  file_count:21875  delete_count:4  deleted_byte_count:347272  replica_placement:100  version:3  modified_at_second:1614877481 
+          volume id:271  size:1076992648  collection:"collection2"  file_count:22640  delete_count:1  deleted_byte_count:30645  replica_placement:100  version:3  modified_at_second:1614877504 
+          volume id:273  size:1074873432  collection:"collection2"  file_count:20511  delete_count:3  deleted_byte_count:46076  replica_placement:100  version:3  modified_at_second:1614884046 
+          volume id:274  size:1075994128  collection:"collection2"  file_count:20997  replica_placement:100  version:3  modified_at_second:1614884113 
+          volume id:276  size:1076899888  collection:"collection2"  file_count:20190  delete_count:1  deleted_byte_count:8798  replica_placement:100  version:3  modified_at_second:1614884003 
+          volume id:277  size:1074956160  collection:"collection2"  file_count:19260  delete_count:2  deleted_byte_count:172356  replica_placement:100  version:3  modified_at_second:1614889988 
+          volume id:279  size:1077325096  collection:"collection2"  file_count:19671  delete_count:6  deleted_byte_count:379116  replica_placement:100  version:3  modified_at_second:1614890230 
+          volume id:282  size:1075232240  collection:"collection2"  file_count:22659  delete_count:4  deleted_byte_count:67915  replica_placement:100  version:3  modified_at_second:1614897304 
+          volume id:284  size:1074533384  collection:"collection2"  file_count:22196  delete_count:4  deleted_byte_count:154683  replica_placement:100  version:3  modified_at_second:1614897231 
+          volume id:285  size:1082128576  collection:"collection2"  file_count:21804  delete_count:1  deleted_byte_count:1064  replica_placement:100  version:3  modified_at_second:1614897165 
+          volume id:286  size:1077464824  collection:"collection2"  file_count:23905  delete_count:6  deleted_byte_count:630577  replica_placement:100  version:3  modified_at_second:1614897401 
+          volume id:287  size:1074590528  collection:"collection2"  file_count:28163  delete_count:5  deleted_byte_count:35727  replica_placement:100  version:3  modified_at_second:1614904874 
+          volume id:288  size:1075406800  collection:"collection2"  file_count:27243  delete_count:2  deleted_byte_count:51519  replica_placement:100  version:3  modified_at_second:1614904738 
+          volume id:292  size:1092010744  collection:"collection2"  file_count:26781  delete_count:5  deleted_byte_count:508910  replica_placement:100  version:3  modified_at_second:1614912327 
+          volume id:293  size:1075409776  collection:"collection2"  file_count:26063  delete_count:4  deleted_byte_count:183834  replica_placement:100  version:3  modified_at_second:1614912235 
+          volume id:294  size:1075443992  collection:"collection2"  file_count:26076  delete_count:4  deleted_byte_count:194914  replica_placement:100  version:3  modified_at_second:1614912220 
+          volume id:295  size:1074702376  collection:"collection2"  file_count:24488  delete_count:3  deleted_byte_count:48555  replica_placement:100  version:3  modified_at_second:1614911929 
+          volume id:300  size:1076212424  collection:"collection2"  file_count:22892  delete_count:2  deleted_byte_count:61320  replica_placement:100  version:3  modified_at_second:1614918464 
+          volume id:304  size:1081038888  collection:"collection2"  file_count:24505  delete_count:2  deleted_byte_count:124447  replica_placement:100  version:3  modified_at_second:1614925567 
+          volume id:305  size:1074185552  collection:"collection2"  file_count:22074  delete_count:5  deleted_byte_count:20221  replica_placement:100  version:3  modified_at_second:1614925312 
+          volume id:310  size:1074761520  collection:"collection2"  file_count:21441  delete_count:3  deleted_byte_count:13934  replica_placement:100  version:3  modified_at_second:1614931077 
+          volume id:311  size:1088248208  collection:"collection2"  file_count:23553  delete_count:6  deleted_byte_count:191716  replica_placement:100  version:3  modified_at_second:1614931460 
+          volume id:312  size:1075037808  collection:"collection2"  file_count:22524  replica_placement:100  version:3  modified_at_second:1614937832 
+          volume id:313  size:1074876016  collection:"collection2"  file_count:22404  delete_count:4  deleted_byte_count:51728  replica_placement:100  version:3  modified_at_second:1614937755 
+          volume id:314  size:1074670840  collection:"collection2"  file_count:20964  delete_count:4  deleted_byte_count:304291  replica_placement:100  version:3  modified_at_second:1614937441 
+          volume id:315  size:1084153456  collection:"collection2"  file_count:23638  delete_count:2  deleted_byte_count:53956  replica_placement:100  version:3  modified_at_second:1614937884 
+          volume id:316  size:1077720784  collection:"collection2"  file_count:22605  delete_count:1  deleted_byte_count:8503  replica_placement:100  version:3  modified_at_second:1614937838 
+          volume id:317  size:1076215040  collection:"collection2"  file_count:23572  delete_count:2  deleted_byte_count:1441356  replica_placement:100  version:3  modified_at_second:1614943965 
+          volume id:319  size:1073952744  collection:"collection2"  file_count:22286  delete_count:2  deleted_byte_count:43421  replica_placement:100  version:3  modified_at_second:1614943810 
+          volume id:320  size:1082437736  collection:"collection2"  file_count:21544  delete_count:3  deleted_byte_count:16712  replica_placement:100  version:3  modified_at_second:1614943591 
+          volume id:321  size:1081477960  collection:"collection2"  file_count:23531  delete_count:5  deleted_byte_count:262564  replica_placement:100  version:3  modified_at_second:1614943982 
+          volume id:322  size:1078471600  collection:"collection2"  file_count:21905  delete_count:3  deleted_byte_count:145002  replica_placement:100  version:3  modified_at_second:1614950574 
+          volume id:324  size:1075606712  collection:"collection2"  file_count:20799  delete_count:1  deleted_byte_count:251210  replica_placement:100  version:3  modified_at_second:1614950310 
+          volume id:326  size:1076059936  collection:"collection2"  file_count:22564  delete_count:2  deleted_byte_count:192886  replica_placement:100  version:3  modified_at_second:1614950619 
+          volume id:327  size:1076121224  collection:"collection2"  file_count:22007  delete_count:3  deleted_byte_count:60358  replica_placement:100  version:3  modified_at_second:1614956487 
+          volume id:328  size:1074767928  collection:"collection2"  file_count:21720  delete_count:3  deleted_byte_count:56429  replica_placement:100  version:3  modified_at_second:1614956362 
+          volume id:329  size:1076691776  collection:"collection2"  file_count:22411  delete_count:5  deleted_byte_count:214092  replica_placement:100  version:3  modified_at_second:1614956485 
+          volume id:331  size:1074957192  collection:"collection2"  file_count:21230  delete_count:4  deleted_byte_count:62145  replica_placement:100  version:3  modified_at_second:1614956259 
+          volume id:333  size:1074270192  collection:"collection2"  file_count:21271  delete_count:2  deleted_byte_count:168122  replica_placement:100  version:3  modified_at_second:1614962697 
+          volume id:335  size:1076235176  collection:"collection2"  file_count:22391  delete_count:3  deleted_byte_count:8838  replica_placement:100  version:3  modified_at_second:1614962970 
+          volume id:336  size:1087853032  collection:"collection2"  file_count:22801  delete_count:2  deleted_byte_count:26394  replica_placement:100  version:3  modified_at_second:1614963003 
+          volume id:338  size:1076118360  collection:"collection2"  file_count:21680  replica_placement:100  version:3  modified_at_second:1614969850 
+          volume id:342  size:1080186296  collection:"collection2"  file_count:22405  delete_count:4  deleted_byte_count:64819  replica_placement:100  version:3  modified_at_second:1614977518 
+          volume id:343  size:1075345184  collection:"collection2"  file_count:21095  delete_count:2  deleted_byte_count:20581  replica_placement:100  version:3  modified_at_second:1614977148 
+          volume id:349  size:1075957824  collection:"collection2"  file_count:22395  delete_count:2  deleted_byte_count:61565  replica_placement:100  version:3  modified_at_second:1614984748 
+          volume id:350  size:1074756688  collection:"collection2"  file_count:21990  delete_count:3  deleted_byte_count:233881  replica_placement:100  version:3  modified_at_second:1614984682 
+          volume id:354  size:1085213992  collection:"collection2"  file_count:23150  delete_count:4  deleted_byte_count:82391  replica_placement:100  version:3  modified_at_second:1614992207 
+          volume id:356  size:1083304992  collection:"collection2"  file_count:21552  delete_count:4  deleted_byte_count:14472  replica_placement:100  version:3  modified_at_second:1614992027 
+          volume id:358  size:1085152312  collection:"collection2"  file_count:23756  delete_count:3  deleted_byte_count:44531  replica_placement:100  version:3  modified_at_second:1614998824 
+          volume id:359  size:1074211240  collection:"collection2"  file_count:22437  delete_count:2  deleted_byte_count:187953  replica_placement:100  version:3  modified_at_second:1614998711 
+          volume id:362  size:1074074120  collection:"collection2"  file_count:20595  delete_count:1  deleted_byte_count:112145  replica_placement:100  version:3  modified_at_second:1615004407 
+          volume id:363  size:1078859496  collection:"collection2"  file_count:23177  delete_count:4  deleted_byte_count:9601  replica_placement:100  version:3  modified_at_second:1615004822 
+          volume id:364  size:1081280816  collection:"collection2"  file_count:22686  delete_count:1  deleted_byte_count:84375  replica_placement:100  version:3  modified_at_second:1615004813 
+          volume id:365  size:1075736632  collection:"collection2"  file_count:22193  delete_count:5  deleted_byte_count:259033  replica_placement:100  version:3  modified_at_second:1615004776 
+          volume id:366  size:1075267272  collection:"collection2"  file_count:21856  delete_count:5  deleted_byte_count:138363  replica_placement:100  version:3  modified_at_second:1615004703 
+          volume id:367  size:1076403648  collection:"collection2"  file_count:22995  delete_count:2  deleted_byte_count:36955  replica_placement:100  version:3  modified_at_second:1615010985 
+          volume id:368  size:1074822016  collection:"collection2"  file_count:22252  delete_count:4  deleted_byte_count:3291946  replica_placement:100  version:3  modified_at_second:1615010878 
+          volume id:369  size:1091472040  collection:"collection2"  file_count:23709  delete_count:4  deleted_byte_count:400876  replica_placement:100  version:3  modified_at_second:1615011019 
+          volume id:370  size:1076040480  collection:"collection2"  file_count:22092  delete_count:2  deleted_byte_count:115388  replica_placement:100  version:3  modified_at_second:1615010877 
+          volume id:371  size:1078806160  collection:"collection2"  file_count:22685  delete_count:2  deleted_byte_count:68905  replica_placement:100  version:3  modified_at_second:1615010994 
+          volume id:372  size:1076193312  collection:"collection2"  file_count:22774  delete_count:1  deleted_byte_count:3495  replica_placement:100  version:3  modified_at_second:1615016911 
+          volume id:374  size:1085011080  collection:"collection2"  file_count:23054  delete_count:2  deleted_byte_count:89034  replica_placement:100  version:3  modified_at_second:1615016917 
+          volume id:375  size:1076140688  collection:"collection2"  file_count:21880  delete_count:2  deleted_byte_count:51103  replica_placement:100  version:3  modified_at_second:1615016787 
+          volume id:378  size:959273824  collection:"collection2"  file_count:15031  replica_placement:100  version:3  modified_at_second:1615632323 
+          volume id:379  size:1014108592  collection:"collection2"  file_count:15360  delete_count:8  deleted_byte_count:2524591  replica_placement:100  version:3  modified_at_second:1615632323 
+          volume id:380  size:1010760464  collection:"collection2"  file_count:14920  delete_count:7  deleted_byte_count:134859  replica_placement:100  version:3  modified_at_second:1615632322 
+        Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
+      DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
+    Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
+  DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 
+  DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0)
+    Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0)
+      DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0)
+        Disk hdd(volume:299/3000 active:299 free:2701 remote:0)
+          volume id:5  size:293806008  file_count:1669  delete_count:2  deleted_byte_count:274334  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631342 
+          volume id:11  size:1109224552  collection:"collection0"  file_count:44  replica_placement:100  version:3  modified_at_second:1615629606 
+          volume id:18  size:1096678688  collection:"collection0"  file_count:88  delete_count:4  deleted_byte_count:8633  replica_placement:100  version:3  modified_at_second:1615631673 
+          volume id:19  size:1096923792  collection:"collection0"  file_count:100  delete_count:10  deleted_byte_count:75779917  replica_placement:100  version:3  compact_revision:4  modified_at_second:1615630117 
+          volume id:20  size:1074760432  collection:"collection0"  file_count:82  delete_count:5  deleted_byte_count:12156  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615629340 
+          volume id:26  size:532657632  collection:"collection0"  file_count:100  delete_count:4  deleted_byte_count:9081  replica_placement:100  version:3  modified_at_second:1614170024 
+          volume id:27  size:298886792  file_count:1608  replica_placement:100  version:3  modified_at_second:1615632482 
+          volume id:28  size:308919192  file_count:1591  delete_count:1  deleted_byte_count:125280  replica_placement:100  version:3  modified_at_second:1615631762 
+          volume id:29  size:281582688  file_count:1537  replica_placement:100  version:3  modified_at_second:1615629422 
+          volume id:30  size:289466144  file_count:1566  delete_count:1  deleted_byte_count:124972  replica_placement:100  version:3  modified_at_second:1615632422 
+          volume id:31  size:273363256  file_count:1498  replica_placement:100  version:3  modified_at_second:1615631642 
+          volume id:32  size:281343360  file_count:1497  replica_placement:100  version:3  modified_at_second:1615632362 
+          volume id:33  size:1130226344  collection:"collection1"  file_count:7322  delete_count:172  deleted_byte_count:45199399  replica_placement:100  version:3  modified_at_second:1615618789 
+          volume id:34  size:1077111136  collection:"collection1"  file_count:9781  delete_count:110  deleted_byte_count:20894827  replica_placement:100  version:3  modified_at_second:1615619366 
+          volume id:35  size:1075241744  collection:"collection1"  file_count:10523  delete_count:97  deleted_byte_count:46586217  replica_placement:100  version:3  modified_at_second:1615618790 
+          volume id:36  size:1075118336  collection:"collection1"  file_count:10341  delete_count:118  deleted_byte_count:24753278  replica_placement:100  version:3  modified_at_second:1615606148 
+          volume id:37  size:1075895576  collection:"collection1"  file_count:12013  delete_count:98  deleted_byte_count:50747932  replica_placement:100  version:3  modified_at_second:1615594776 
+          volume id:38  size:1075545744  collection:"collection1"  file_count:13324  delete_count:100  deleted_byte_count:25223906  replica_placement:100  version:3  modified_at_second:1615569830 
+          volume id:39  size:1076606536  collection:"collection1"  file_count:12612  delete_count:78  deleted_byte_count:17462730  replica_placement:100  version:3  modified_at_second:1615611959 
+          volume id:40  size:1075358552  collection:"collection1"  file_count:12597  delete_count:62  deleted_byte_count:11657901  replica_placement:100  version:3  modified_at_second:1615612994 
+          volume id:41  size:1076283592  collection:"collection1"  file_count:12088  delete_count:84  deleted_byte_count:19311237  replica_placement:100  version:3  modified_at_second:1615596736 
+          volume id:42  size:1093948352  collection:"collection1"  file_count:7889  delete_count:47  deleted_byte_count:5697275  replica_placement:100  version:3  modified_at_second:1615548906 
+          volume id:43  size:1116445864  collection:"collection1"  file_count:7355  delete_count:57  deleted_byte_count:9727158  replica_placement:100  version:3  modified_at_second:1615566167 
+          volume id:44  size:1077582560  collection:"collection1"  file_count:7295  delete_count:50  deleted_byte_count:12618414  replica_placement:100  version:3  modified_at_second:1615566170 
+          volume id:45  size:1075254640  collection:"collection1"  file_count:10772  delete_count:76  deleted_byte_count:22426345  replica_placement:100  version:3  modified_at_second:1615573498 
+          volume id:46  size:1075286056  collection:"collection1"  file_count:9947  delete_count:309  deleted_byte_count:105601163  replica_placement:100  version:3  modified_at_second:1615569825 
+          volume id:47  size:444599784  collection:"collection1"  file_count:709  delete_count:19  deleted_byte_count:11913451  replica_placement:100  version:3  modified_at_second:1615632397 
+          volume id:48  size:1076778664  collection:"collection1"  file_count:9850  delete_count:77  deleted_byte_count:16641907  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630690 
+          volume id:49  size:1078775288  collection:"collection1"  file_count:9631  delete_count:27  deleted_byte_count:5985628  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615575823 
+          volume id:50  size:1076688288  collection:"collection1"  file_count:7921  delete_count:26  deleted_byte_count:5162032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615610876 
+          volume id:51  size:1076796120  collection:"collection1"  file_count:10550  delete_count:39  deleted_byte_count:12723654  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615547786 
+          volume id:53  size:1063089216  collection:"collection1"  file_count:9832  delete_count:31  deleted_byte_count:9273066  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
+          volume id:54  size:1045022288  collection:"collection1"  file_count:9409  delete_count:29  deleted_byte_count:15102818  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630813 
+          volume id:55  size:1012890016  collection:"collection1"  file_count:8651  delete_count:27  deleted_byte_count:9418841  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631453 
+          volume id:56  size:1002412240  collection:"collection1"  file_count:8762  delete_count:40  deleted_byte_count:65885831  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632014 
+          volume id:57  size:839849792  collection:"collection1"  file_count:7514  delete_count:24  deleted_byte_count:6228543  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631775 
+          volume id:58  size:908064192  collection:"collection1"  file_count:8128  delete_count:21  deleted_byte_count:6113731  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632343 
+          volume id:59  size:988302272  collection:"collection1"  file_count:8098  delete_count:20  deleted_byte_count:3947615  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632238 
+          volume id:60  size:1010702480  collection:"collection1"  file_count:8969  delete_count:79  deleted_byte_count:24782814  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632439 
+          volume id:61  size:975604544  collection:"collection1"  file_count:8683  delete_count:20  deleted_byte_count:10276072  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631176 
+          volume id:62  size:873845904  collection:"collection1"  file_count:7897  delete_count:23  deleted_byte_count:10920170  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631133 
+          volume id:63  size:956941176  collection:"collection1"  file_count:8271  delete_count:32  deleted_byte_count:15876189  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632036 
+          volume id:64  size:965638424  collection:"collection1"  file_count:8218  delete_count:27  deleted_byte_count:6922489  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631032 
+          volume id:65  size:823283608  collection:"collection1"  file_count:7834  delete_count:29  deleted_byte_count:5950610  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
+          volume id:66  size:821343440  collection:"collection1"  file_count:7383  delete_count:29  deleted_byte_count:12010343  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631968 
+          volume id:67  size:878713880  collection:"collection1"  file_count:7299  delete_count:117  deleted_byte_count:24857326  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632156 
+          volume id:69  size:863913896  collection:"collection1"  file_count:7291  delete_count:100  deleted_byte_count:25335024  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630534 
+          volume id:70  size:886695472  collection:"collection1"  file_count:7769  delete_count:164  deleted_byte_count:45162513  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632398 
+          volume id:71  size:907608392  collection:"collection1"  file_count:7658  delete_count:122  deleted_byte_count:27622941  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632307 
+          volume id:72  size:903990720  collection:"collection1"  file_count:6996  delete_count:240  deleted_byte_count:74147727  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630985 
+          volume id:73  size:929047544  collection:"collection1"  file_count:7038  delete_count:227  deleted_byte_count:65336664  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630707 
+          volume id:75  size:908045000  collection:"collection1"  file_count:6911  delete_count:268  deleted_byte_count:73934373  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632430 
+          volume id:76  size:985296744  collection:"collection1"  file_count:6566  delete_count:61  deleted_byte_count:44464430  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632284 
+          volume id:77  size:929398296  collection:"collection1"  file_count:7427  delete_count:238  deleted_byte_count:59581579  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632014 
+          volume id:78  size:1075671512  collection:"collection1"  file_count:7540  delete_count:258  deleted_byte_count:71726846  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615582829 
+          volume id:79  size:948225472  collection:"collection1"  file_count:6997  delete_count:227  deleted_byte_count:60625763  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631326 
+          volume id:80  size:827912928  collection:"collection1"  file_count:6916  delete_count:15  deleted_byte_count:5611604  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631159 
+          volume id:81  size:880693168  collection:"collection1"  file_count:7481  delete_count:238  deleted_byte_count:80880878  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631395 
+          volume id:82  size:1041660512  collection:"collection1"  file_count:7043  delete_count:207  deleted_byte_count:52275724  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632430 
+          volume id:83  size:936194288  collection:"collection1"  file_count:7593  delete_count:13  deleted_byte_count:4633917  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632029 
+          volume id:84  size:871262320  collection:"collection1"  file_count:8190  delete_count:14  deleted_byte_count:3150948  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631161 
+          volume id:86  size:1009434632  collection:"collection1"  file_count:8474  delete_count:236  deleted_byte_count:64543674  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630812 
+          volume id:87  size:922274624  collection:"collection1"  file_count:12902  delete_count:13  deleted_byte_count:3412959  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615632438 
+          volume id:88  size:1073767976  collection:"collection1"  file_count:14994  delete_count:207  deleted_byte_count:82380696  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615541383 
+          volume id:89  size:1044421824  collection:"collection1"  file_count:14943  delete_count:243  deleted_byte_count:58543159  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615632208 
+          volume id:90  size:891163760  collection:"collection1"  file_count:14608  delete_count:10  deleted_byte_count:2564369  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615629392 
+          volume id:91  size:936573952  collection:"collection1"  file_count:14686  delete_count:11  deleted_byte_count:4717727  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631851 
+          volume id:92  size:992439144  collection:"collection1"  file_count:7061  delete_count:195  deleted_byte_count:60649573  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630566 
+          volume id:93  size:1079602592  collection:"collection1"  file_count:7878  delete_count:270  deleted_byte_count:74150048  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615556013 
+          volume id:94  size:1030684704  collection:"collection1"  file_count:7660  delete_count:207  deleted_byte_count:70150733  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631616 
+          volume id:95  size:990877824  collection:"collection1"  file_count:6620  delete_count:206  deleted_byte_count:60363604  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615631867 
+          volume id:96  size:989294848  collection:"collection1"  file_count:7544  delete_count:229  deleted_byte_count:59931853  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630778 
+          volume id:98  size:1077836472  collection:"collection1"  file_count:7605  delete_count:202  deleted_byte_count:73180379  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615523691 
+          volume id:99  size:1071718496  collection:"collection1"  file_count:7470  delete_count:8  deleted_byte_count:9624950  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615631175 
+          volume id:100  size:1083617472  collection:"collection1"  file_count:7018  delete_count:187  deleted_byte_count:61304236  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615505914 
+          volume id:101  size:1077109408  collection:"collection1"  file_count:7706  delete_count:226  deleted_byte_count:77864780  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615630994 
+          volume id:102  size:1074359920  collection:"collection1"  file_count:7338  delete_count:7  deleted_byte_count:6499151  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615626682 
+          volume id:103  size:1075863904  collection:"collection1"  file_count:7184  delete_count:186  deleted_byte_count:58872238  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615628417 
+          volume id:106  size:1075458680  collection:"collection1"  file_count:7182  delete_count:307  deleted_byte_count:69349053  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615598137 
+          volume id:107  size:1073811776  collection:"collection1"  file_count:7436  delete_count:168  deleted_byte_count:57747428  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615293569 
+          volume id:108  size:1074648024  collection:"collection1"  file_count:7472  delete_count:194  deleted_byte_count:70864699  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593231 
+          volume id:109  size:1075254560  collection:"collection1"  file_count:7556  delete_count:263  deleted_byte_count:55155265  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615502487 
+          volume id:110  size:1076575744  collection:"collection1"  file_count:6996  delete_count:163  deleted_byte_count:52954032  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615590786 
+          volume id:111  size:1073826176  collection:"collection1"  file_count:7355  delete_count:155  deleted_byte_count:50083578  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615593232 
+          volume id:112  size:1076392512  collection:"collection1"  file_count:8291  delete_count:156  deleted_byte_count:74120183  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569823 
+          volume id:113  size:1076709184  collection:"collection1"  file_count:9355  delete_count:177  deleted_byte_count:59796765  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615569822 
+          volume id:114  size:1074762792  collection:"collection1"  file_count:8802  delete_count:156  deleted_byte_count:38470055  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615591826 
+          volume id:115  size:1076192296  collection:"collection1"  file_count:7690  delete_count:154  deleted_byte_count:32267193  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615285296 
+          volume id:117  size:1073917192  collection:"collection1"  file_count:9520  delete_count:114  deleted_byte_count:21835126  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615573712 
+          volume id:118  size:1074064344  collection:"collection1"  file_count:8738  delete_count:15  deleted_byte_count:3460697  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615516264 
+          volume id:120  size:1076115928  collection:"collection1"  file_count:9639  delete_count:118  deleted_byte_count:33357871  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615482567 
+          volume id:121  size:1078803320  collection:"collection1"  file_count:10113  delete_count:441  deleted_byte_count:94128627  replica_placement:100  version:3  modified_at_second:1615506626 
+          volume id:122  size:1076235312  collection:"collection1"  file_count:9106  delete_count:252  deleted_byte_count:93041272  replica_placement:100  version:3  modified_at_second:1615585912 
+          volume id:123  size:1080491112  collection:"collection1"  file_count:10623  delete_count:302  deleted_byte_count:83956998  replica_placement:100  version:3  modified_at_second:1615585916 
+          volume id:124  size:1074519360  collection:"collection1"  file_count:9457  delete_count:286  deleted_byte_count:74752459  replica_placement:100  version:3  modified_at_second:1615585913 
+          volume id:125  size:1088687040  collection:"collection1"  file_count:9518  delete_count:281  deleted_byte_count:76037905  replica_placement:100  version:3  modified_at_second:1615585913 
+          volume id:126  size:1073867408  collection:"collection1"  file_count:9320  delete_count:278  deleted_byte_count:94547424  replica_placement:100  version:3  modified_at_second:1615585911 
+          volume id:127  size:1074907336  collection:"collection1"  file_count:9900  delete_count:133  deleted_byte_count:48570820  replica_placement:100  version:3  modified_at_second:1615612990 
+          volume id:128  size:1074874632  collection:"collection1"  file_count:9821  delete_count:148  deleted_byte_count:43633334  replica_placement:100  version:3  modified_at_second:1615602670 
+          volume id:129  size:1074704328  collection:"collection1"  file_count:10012  delete_count:150  deleted_byte_count:64491721  replica_placement:100  version:3  modified_at_second:1615627566 
+          volume id:130  size:1075000632  collection:"collection1"  file_count:10633  delete_count:161  deleted_byte_count:34768201  replica_placement:100  version:3  modified_at_second:1615582327 
+          volume id:131  size:1075279584  collection:"collection1"  file_count:10075  delete_count:135  deleted_byte_count:29795712  replica_placement:100  version:3  modified_at_second:1615523898 
+          volume id:132  size:1088539552  collection:"collection1"  file_count:11051  delete_count:71  deleted_byte_count:17178322  replica_placement:100  version:3  modified_at_second:1615619581 
+          volume id:134  size:1074367304  collection:"collection1"  file_count:10662  delete_count:69  deleted_byte_count:25530139  replica_placement:100  version:3  modified_at_second:1615555873 
+          volume id:135  size:1073906776  collection:"collection1"  file_count:10446  delete_count:71  deleted_byte_count:28599975  replica_placement:100  version:3  modified_at_second:1615569816 
+          volume id:136  size:1074433552  collection:"collection1"  file_count:9593  delete_count:72  deleted_byte_count:26912512  replica_placement:100  version:3  modified_at_second:1615376036 
+          volume id:137  size:1074309264  collection:"collection1"  file_count:9633  delete_count:50  deleted_byte_count:27487972  replica_placement:100  version:3  modified_at_second:1615572231 
+          volume id:138  size:1074465744  collection:"collection1"  file_count:10120  delete_count:55  deleted_byte_count:15875438  replica_placement:100  version:3  modified_at_second:1615572231 
+          volume id:140  size:1076203744  collection:"collection1"  file_count:11219  delete_count:57  deleted_byte_count:19864498  replica_placement:100  version:3  modified_at_second:1615571947 
+          volume id:141  size:1074619488  collection:"collection1"  file_count:9840  delete_count:45  deleted_byte_count:40890181  replica_placement:100  version:3  modified_at_second:1615630994 
+          volume id:142  size:1075733064  collection:"collection1"  file_count:9009  delete_count:48  deleted_byte_count:9912854  replica_placement:100  version:3  modified_at_second:1615598913 
+          volume id:143  size:1075011280  collection:"collection1"  file_count:9608  delete_count:51  deleted_byte_count:37282460  replica_placement:100  version:3  modified_at_second:1615488584 
+          volume id:144  size:1074549720  collection:"collection1"  file_count:8780  delete_count:50  deleted_byte_count:52475146  replica_placement:100  version:3  modified_at_second:1615573451 
+          volume id:145  size:1074394928  collection:"collection1"  file_count:9255  delete_count:34  deleted_byte_count:38011392  replica_placement:100  version:3  modified_at_second:1615591825 
+          volume id:146  size:1076337576  collection:"collection1"  file_count:10492  delete_count:50  deleted_byte_count:17071505  replica_placement:100  version:3  modified_at_second:1615632005 
+          volume id:147  size:1077130576  collection:"collection1"  file_count:10451  delete_count:27  deleted_byte_count:8290907  replica_placement:100  version:3  modified_at_second:1615604115 
+          volume id:148  size:1076066568  collection:"collection1"  file_count:9547  delete_count:33  deleted_byte_count:7034089  replica_placement:100  version:3  modified_at_second:1615586390 
+          volume id:149  size:1074989016  collection:"collection1"  file_count:8352  delete_count:35  deleted_byte_count:7179742  replica_placement:100  version:3  modified_at_second:1615494494 
+          volume id:150  size:1076290328  collection:"collection1"  file_count:9328  delete_count:33  deleted_byte_count:43417791  replica_placement:100  version:3  modified_at_second:1615611567 
+          volume id:152  size:1075941400  collection:"collection1"  file_count:9951  delete_count:36  deleted_byte_count:25348335  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:153  size:1078539784  collection:"collection1"  file_count:10924  delete_count:34  deleted_byte_count:12603081  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:154  size:1081244696  collection:"collection1"  file_count:11002  delete_count:31  deleted_byte_count:8467560  replica_placement:100  version:3  modified_at_second:1615478469 
+          volume id:155  size:1075140688  collection:"collection1"  file_count:10882  delete_count:32  deleted_byte_count:10076804  replica_placement:100  version:3  modified_at_second:1615606614 
+          volume id:156  size:1074975832  collection:"collection1"  file_count:9535  delete_count:40  deleted_byte_count:11426621  replica_placement:100  version:3  modified_at_second:1615628341 
+          volume id:157  size:1076758536  collection:"collection1"  file_count:10012  delete_count:19  deleted_byte_count:11688737  replica_placement:100  version:3  modified_at_second:1615597782 
+          volume id:158  size:1087251976  collection:"collection1"  file_count:9972  delete_count:20  deleted_byte_count:10328429  replica_placement:100  version:3  modified_at_second:1615588879 
+          volume id:159  size:1074132368  collection:"collection1"  file_count:9382  delete_count:27  deleted_byte_count:11474574  replica_placement:100  version:3  modified_at_second:1615593593 
+          volume id:160  size:1075680952  collection:"collection1"  file_count:9772  delete_count:22  deleted_byte_count:4981968  replica_placement:100  version:3  modified_at_second:1615597780 
+          volume id:162  size:1074286880  collection:"collection1"  file_count:11220  delete_count:17  deleted_byte_count:1815547  replica_placement:100  version:3  modified_at_second:1615478126 
+          volume id:163  size:1074457192  collection:"collection1"  file_count:12524  delete_count:27  deleted_byte_count:6359619  replica_placement:100  version:3  modified_at_second:1615579313 
+          volume id:164  size:1074261248  collection:"collection1"  file_count:11922  delete_count:25  deleted_byte_count:2923288  replica_placement:100  version:3  modified_at_second:1615620084 
+          volume id:165  size:1073891016  collection:"collection1"  file_count:9152  delete_count:12  deleted_byte_count:19164659  replica_placement:100  version:3  modified_at_second:1615471907 
+          volume id:166  size:1075637536  collection:"collection1"  file_count:14211  delete_count:24  deleted_byte_count:20415490  replica_placement:100  version:3  modified_at_second:1615491019 
+          volume id:168  size:1074718808  collection:"collection1"  file_count:25702  delete_count:40  deleted_byte_count:4024775  replica_placement:100  version:3  modified_at_second:1615585664 
+          volume id:169  size:1073863128  collection:"collection1"  file_count:25248  delete_count:43  deleted_byte_count:3013817  replica_placement:100  version:3  modified_at_second:1615569832 
+          volume id:170  size:1075747096  collection:"collection1"  file_count:24596  delete_count:41  deleted_byte_count:3494711  replica_placement:100  version:3  modified_at_second:1615579204 
+          volume id:171  size:1081881312  collection:"collection1"  file_count:24215  delete_count:36  deleted_byte_count:3191335  replica_placement:100  version:3  modified_at_second:1615596485 
+          volume id:172  size:1074787312  collection:"collection1"  file_count:31236  delete_count:50  deleted_byte_count:3316482  replica_placement:100  version:3  modified_at_second:1615612385 
+          volume id:173  size:1074154648  collection:"collection1"  file_count:30884  delete_count:34  deleted_byte_count:2430948  replica_placement:100  version:3  modified_at_second:1615591904 
+          volume id:175  size:1077742504  collection:"collection1"  file_count:32353  delete_count:33  deleted_byte_count:1861403  replica_placement:100  version:3  modified_at_second:1615559515 
+          volume id:176  size:1073854800  collection:"collection1"  file_count:30582  delete_count:34  deleted_byte_count:7701976  replica_placement:100  version:3  modified_at_second:1615626169 
+          volume id:177  size:1074120120  collection:"collection1"  file_count:22293  delete_count:16  deleted_byte_count:3719562  replica_placement:100  version:3  modified_at_second:1615516891 
+          volume id:178  size:1087560112  collection:"collection1"  file_count:23482  delete_count:22  deleted_byte_count:18810492  replica_placement:100  version:3  modified_at_second:1615541369 
+          volume id:180  size:1078438536  collection:"collection1"  file_count:23614  delete_count:12  deleted_byte_count:4496474  replica_placement:100  version:3  modified_at_second:1614773242 
+          volume id:181  size:1074571768  collection:"collection1"  file_count:22898  delete_count:19  deleted_byte_count:6628413  replica_placement:100  version:3  modified_at_second:1614745116 
+          volume id:182  size:1076131280  collection:"collection1"  file_count:31987  delete_count:21  deleted_byte_count:1416142  replica_placement:100  version:3  modified_at_second:1615568922 
+          volume id:183  size:1076361448  collection:"collection1"  file_count:31293  delete_count:16  deleted_byte_count:468841  replica_placement:100  version:3  modified_at_second:1615572982 
+          volume id:184  size:1074594160  collection:"collection1"  file_count:31368  delete_count:22  deleted_byte_count:857453  replica_placement:100  version:3  modified_at_second:1615586578 
+          volume id:185  size:1074099624  collection:"collection1"  file_count:30612  delete_count:17  deleted_byte_count:2610847  replica_placement:100  version:3  modified_at_second:1615506832 
+          volume id:186  size:1074220864  collection:"collection1"  file_count:31450  delete_count:15  deleted_byte_count:391855  replica_placement:100  version:3  modified_at_second:1615614933 
+          volume id:187  size:1074395944  collection:"collection1"  file_count:31853  delete_count:17  deleted_byte_count:454283  replica_placement:100  version:3  modified_at_second:1615590490 
+          volume id:188  size:1074732792  collection:"collection1"  file_count:31867  delete_count:19  deleted_byte_count:393743  replica_placement:100  version:3  modified_at_second:1615487645 
+          volume id:189  size:1074847896  collection:"collection1"  file_count:31450  delete_count:16  deleted_byte_count:1040552  replica_placement:100  version:3  modified_at_second:1615335661 
+          volume id:190  size:1074008912  collection:"collection1"  file_count:31987  delete_count:11  deleted_byte_count:685125  replica_placement:100  version:3  modified_at_second:1615447161 
+          volume id:191  size:1075493024  collection:"collection1"  file_count:31301  delete_count:19  deleted_byte_count:708401  replica_placement:100  version:3  modified_at_second:1615357456 
+          volume id:192  size:1075857400  collection:"collection1"  file_count:31490  delete_count:25  deleted_byte_count:720617  replica_placement:100  version:3  modified_at_second:1615621632 
+          volume id:193  size:1076616768  collection:"collection1"  file_count:31907  delete_count:16  deleted_byte_count:464900  replica_placement:100  version:3  modified_at_second:1615507875 
+          volume id:194  size:1073985624  collection:"collection1"  file_count:31434  delete_count:18  deleted_byte_count:391432  replica_placement:100  version:3  modified_at_second:1615559499 
+          volume id:195  size:1074158312  collection:"collection1"  file_count:31453  delete_count:15  deleted_byte_count:718266  replica_placement:100  version:3  modified_at_second:1615559331 
+          volume id:196  size:1074594784  collection:"collection1"  file_count:31665  delete_count:18  deleted_byte_count:3468922  replica_placement:100  version:3  modified_at_second:1615501688 
+          volume id:197  size:1075423296  collection:"collection1"  file_count:16473  delete_count:15  deleted_byte_count:12552442  replica_placement:100  version:3  modified_at_second:1615485253 
+          volume id:198  size:1075104712  collection:"collection1"  file_count:16577  delete_count:18  deleted_byte_count:6583181  replica_placement:100  version:3  modified_at_second:1615623369 
+          volume id:199  size:1078117688  collection:"collection1"  file_count:16497  delete_count:14  deleted_byte_count:1514286  replica_placement:100  version:3  modified_at_second:1615585984 
+          volume id:200  size:1075630536  collection:"collection1"  file_count:16380  delete_count:18  deleted_byte_count:1103109  replica_placement:100  version:3  modified_at_second:1615485252 
+          volume id:201  size:1091460440  collection:"collection1"  file_count:16684  delete_count:26  deleted_byte_count:5590335  replica_placement:100  version:3  modified_at_second:1615585987 
+          volume id:202  size:1077533160  collection:"collection1"  file_count:2847  delete_count:67  deleted_byte_count:65172985  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615588497 
+          volume id:203  size:1027316272  collection:"collection1"  file_count:3040  delete_count:11  deleted_byte_count:3993230  replica_placement:100  version:3  compact_revision:3  modified_at_second:1615631728 
+          volume id:204  size:1079766872  collection:"collection1"  file_count:3233  delete_count:255  deleted_byte_count:104707641  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615565701 
+          volume id:205  size:1078485304  collection:"collection1"  file_count:2869  delete_count:43  deleted_byte_count:18290259  replica_placement:100  version:3  compact_revision:2  modified_at_second:1615579314 
+          volume id:206  size:1082045848  collection:"collection1"  file_count:2979  delete_count:225  deleted_byte_count:88220074  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630989 
+          volume id:207  size:1081939960  collection:"collection1"  file_count:3010  delete_count:4  deleted_byte_count:692350  replica_placement:100  version:3  modified_at_second:1615269061 
+          volume id:208  size:1077863624  collection:"collection1"  file_count:3147  delete_count:6  deleted_byte_count:858726  replica_placement:100  version:3  modified_at_second:1615495515 
+          volume id:210  size:1094311304  collection:"collection1"  file_count:3468  delete_count:4  deleted_byte_count:466433  replica_placement:100  version:3  modified_at_second:1615495515 
+          volume id:212  size:1078293448  collection:"collection1"  file_count:3106  delete_count:6  deleted_byte_count:2085755  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:213  size:1093588072  collection:"collection1"  file_count:3681  delete_count:12  deleted_byte_count:3138791  replica_placement:100  version:3  modified_at_second:1615586387 
+          volume id:214  size:1074486992  collection:"collection1"  file_count:3217  delete_count:10  deleted_byte_count:6392871  replica_placement:100  version:3  modified_at_second:1615586383 
+          volume id:215  size:1074798704  collection:"collection1"  file_count:2819  delete_count:31  deleted_byte_count:10873569  replica_placement:100  version:3  modified_at_second:1615586386 
+          volume id:217  size:1075381872  collection:"collection1"  file_count:3331  delete_count:14  deleted_byte_count:2009141  replica_placement:100  version:3  modified_at_second:1615401638 
+          volume id:218  size:1081263944  collection:"collection1"  file_count:3433  delete_count:14  deleted_byte_count:3454237  replica_placement:100  version:3  modified_at_second:1615603637 
+          volume id:219  size:1092298816  collection:"collection1"  file_count:3193  delete_count:17  deleted_byte_count:2047576  replica_placement:100  version:3  modified_at_second:1615579316 
+          volume id:220  size:1081928312  collection:"collection1"  file_count:3166  delete_count:13  deleted_byte_count:4127709  replica_placement:100  version:3  modified_at_second:1615579317 
+          volume id:221  size:1106545456  collection:"collection1"  file_count:3153  delete_count:11  deleted_byte_count:1496835  replica_placement:100  version:3  modified_at_second:1615269138 
+          volume id:222  size:1106623104  collection:"collection1"  file_count:3273  delete_count:11  deleted_byte_count:2114627  replica_placement:100  version:3  modified_at_second:1615586243 
+          volume id:223  size:1075233064  collection:"collection1"  file_count:2966  delete_count:9  deleted_byte_count:744001  replica_placement:100  version:3  modified_at_second:1615586244 
+          volume id:224  size:1093691520  collection:"collection1"  file_count:3463  delete_count:10  deleted_byte_count:1128328  replica_placement:100  version:3  modified_at_second:1615601870 
+          volume id:225  size:1080698928  collection:"collection1"  file_count:3115  delete_count:7  deleted_byte_count:18170416  replica_placement:100  version:3  modified_at_second:1615434684 
+          volume id:226  size:1103504768  collection:"collection1"  file_count:2965  delete_count:10  deleted_byte_count:2639254  replica_placement:100  version:3  modified_at_second:1615601867 
+          volume id:228  size:1109784072  collection:"collection1"  file_count:2504  delete_count:24  deleted_byte_count:5458950  replica_placement:100  version:3  modified_at_second:1615610489 
+          volume id:230  size:1080722984  collection:"collection1"  file_count:2898  delete_count:15  deleted_byte_count:3929261  replica_placement:100  version:3  modified_at_second:1615610537 
+          volume id:232  size:1073901520  collection:"collection1"  file_count:3004  delete_count:54  deleted_byte_count:10273081  replica_placement:100  version:3  modified_at_second:1615611351 
+          volume id:234  size:1073835280  collection:"collection1"  file_count:2965  delete_count:41  deleted_byte_count:4960354  replica_placement:100  version:3  modified_at_second:1615611351 
+          volume id:235  size:1075586104  collection:"collection1"  file_count:2767  delete_count:33  deleted_byte_count:3216540  replica_placement:100  version:3  modified_at_second:1615611354 
+          volume id:236  size:1089476136  collection:"collection1"  file_count:3231  delete_count:53  deleted_byte_count:11625921  replica_placement:100  version:3  modified_at_second:1615611351 
+          volume id:237  size:375722792  collection:"collection1"  file_count:736  delete_count:16  deleted_byte_count:4464870  replica_placement:100  version:3  modified_at_second:1615631727 
+          volume id:238  size:354320000  collection:"collection1"  file_count:701  delete_count:17  deleted_byte_count:5940420  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615632030 
+          volume id:239  size:426569024  collection:"collection1"  file_count:693  delete_count:19  deleted_byte_count:13020783  replica_placement:100  version:3  compact_revision:1  modified_at_second:1615630841 
+          volume id:240  size:424791528  collection:"collection1"  file_count:733  delete_count:13  deleted_byte_count:7515220  replica_placement:100  version:3  modified_at_second:1615631670 
+          volume id:241  size:380217424  collection:"collection1"  file_count:633  delete_count:6  deleted_byte_count:1715768  replica_placement:100  version:3  modified_at_second:1615632006 
+          volume id:242  size:1075383392  collection:"collection2"  file_count:10470  replica_placement:100  version:3  modified_at_second:1614852116 
+          volume id:243  size:1088174704  collection:"collection2"  file_count:11109  delete_count:1  deleted_byte_count:938  replica_placement:100  version:3  modified_at_second:1614852203 
+          volume id:244  size:1080295352  collection:"collection2"  file_count:10812  delete_count:1  deleted_byte_count:795  replica_placement:100  version:3  modified_at_second:1615628825 
+          volume id:246  size:1075998672  collection:"collection2"  file_count:10365  delete_count:1  deleted_byte_count:13112  replica_placement:100  version:3  modified_at_second:1614852106 
+          volume id:247  size:1075859808  collection:"collection2"  file_count:10443  delete_count:2  deleted_byte_count:564486  replica_placement:100  version:3  modified_at_second:1614856152 
+          volume id:248  size:1084301208  collection:"collection2"  file_count:11217  delete_count:4  deleted_byte_count:746488  replica_placement:100  version:3  modified_at_second:1614856285 
+          volume id:250  size:1080572168  collection:"collection2"  file_count:10220  replica_placement:100  version:3  modified_at_second:1614856129 
+          volume id:252  size:1075065264  collection:"collection2"  file_count:14622  delete_count:2  deleted_byte_count:5228  replica_placement:100  version:3  modified_at_second:1614861200 
+          volume id:253  size:1087328880  collection:"collection2"  file_count:14920  delete_count:3  deleted_byte_count:522994  replica_placement:100  version:3  modified_at_second:1614861258 
+          volume id:254  size:1074830736  collection:"collection2"  file_count:14140  delete_count:2  deleted_byte_count:105892  replica_placement:100  version:3  modified_at_second:1614861115 
+          volume id:255  size:1079581640  collection:"collection2"  file_count:14877  delete_count:3  deleted_byte_count:101223  replica_placement:100  version:3  modified_at_second:1614861233 
+          volume id:256  size:1074283592  collection:"collection2"  file_count:14157  delete_count:1  deleted_byte_count:18156  replica_placement:100  version:3  modified_at_second:1614861100 
+          volume id:257  size:1082621720  collection:"collection2"  file_count:18172  delete_count:2  deleted_byte_count:25125  replica_placement:100  version:3  modified_at_second:1614866402 
+          volume id:258  size:1075527216  collection:"collection2"  file_count:18421  delete_count:4  deleted_byte_count:267833  replica_placement:100  version:3  modified_at_second:1614866420 
+          volume id:259  size:1075507848  collection:"collection2"  file_count:18079  delete_count:2  deleted_byte_count:71992  replica_placement:100  version:3  modified_at_second:1614866381 
+          volume id:260  size:1075105664  collection:"collection2"  file_count:17316  delete_count:4  deleted_byte_count:2015310  replica_placement:100  version:3  modified_at_second:1614866226 
+          volume id:261  size:1076628592  collection:"collection2"  file_count:18355  delete_count:1  deleted_byte_count:1155  replica_placement:100  version:3  modified_at_second:1614866420 
+          volume id:262  size:1078492584  collection:"collection2"  file_count:20390  delete_count:3  deleted_byte_count:287601  replica_placement:100  version:3  modified_at_second:1614871601 
+          volume id:264  size:1081624192  collection:"collection2"  file_count:21151  replica_placement:100  version:3  modified_at_second:1614871629 
+          volume id:265  size:1076401104  collection:"collection2"  file_count:19932  delete_count:2  deleted_byte_count:160823  replica_placement:100  version:3  modified_at_second:1614871543 
+          volume id:266  size:1075617552  collection:"collection2"  file_count:20075  delete_count:1  deleted_byte_count:1039  replica_placement:100  version:3  modified_at_second:1614871526 
+          volume id:267  size:1075699376  collection:"collection2"  file_count:21039  delete_count:3  deleted_byte_count:59956  replica_placement:100  version:3  modified_at_second:1614877294 
+          volume id:270  size:1076876424  collection:"collection2"  file_count:22057  delete_count:1  deleted_byte_count:43916  replica_placement:100  version:3  modified_at_second:1614877469 
+          volume id:271  size:1076992704  collection:"collection2"  file_count:22640  delete_count:1  deleted_byte_count:30645  replica_placement:100  version:3  modified_at_second:1614877504 
+          volume id:272  size:1076145912  collection:"collection2"  file_count:21034  delete_count:2  deleted_byte_count:216564  replica_placement:100  version:3  modified_at_second:1614884139 
+          volume id:273  size:1074873432  collection:"collection2"  file_count:20511  delete_count:3  deleted_byte_count:46076  replica_placement:100  version:3  modified_at_second:1614884046 
+          volume id:274  size:1075994184  collection:"collection2"  file_count:20997  replica_placement:100  version:3  modified_at_second:1614884113 
+          volume id:275  size:1078349024  collection:"collection2"  file_count:20808  delete_count:1  deleted_byte_count:1118  replica_placement:100  version:3  modified_at_second:1614884147 
+          volume id:276  size:1076899880  collection:"collection2"  file_count:20190  delete_count:1  deleted_byte_count:8798  replica_placement:100  version:3  modified_at_second:1614884003 
+          volume id:278  size:1078798632  collection:"collection2"  file_count:20597  delete_count:5  deleted_byte_count:400060  replica_placement:100  version:3  modified_at_second:1614890292 
+          volume id:280  size:1077432160  collection:"collection2"  file_count:20286  delete_count:1  deleted_byte_count:879  replica_placement:100  version:3  modified_at_second:1614890262 
+          volume id:281  size:1077581064  collection:"collection2"  file_count:20206  delete_count:3  deleted_byte_count:143964  replica_placement:100  version:3  modified_at_second:1614890237 
+          volume id:282  size:1075232184  collection:"collection2"  file_count:22659  delete_count:4  deleted_byte_count:67915  replica_placement:100  version:3  modified_at_second:1614897304 
+          volume id:283  size:1080178880  collection:"collection2"  file_count:19462  delete_count:7  deleted_byte_count:660407  replica_placement:100  version:3  modified_at_second:1614896623 
+          volume id:286  size:1077464816  collection:"collection2"  file_count:23905  delete_count:6  deleted_byte_count:630577  replica_placement:100  version:3  modified_at_second:1614897401 
+          volume id:287  size:1074590536  collection:"collection2"  file_count:28163  delete_count:5  deleted_byte_count:35727  replica_placement:100  version:3  modified_at_second:1614904875 
+          volume id:288  size:1075406920  collection:"collection2"  file_count:27243  delete_count:2  deleted_byte_count:51519  replica_placement:100  version:3  modified_at_second:1614904738 
+          volume id:289  size:1075284312  collection:"collection2"  file_count:29342  delete_count:5  deleted_byte_count:100454  replica_placement:100  version:3  modified_at_second:1614904977 
+          volume id:290  size:1074723800  collection:"collection2"  file_count:28340  delete_count:4  deleted_byte_count:199064  replica_placement:100  version:3  modified_at_second:1614904924 
+          volume id:292  size:1092010672  collection:"collection2"  file_count:26781  delete_count:5  deleted_byte_count:508910  replica_placement:100  version:3  modified_at_second:1614912325 
+          volume id:295  size:1074702320  collection:"collection2"  file_count:24488  delete_count:3  deleted_byte_count:48555  replica_placement:100  version:3  modified_at_second:1614911929 
+          volume id:296  size:1077824056  collection:"collection2"  file_count:26741  delete_count:4  deleted_byte_count:199906  replica_placement:100  version:3  modified_at_second:1614912301 
+          volume id:297  size:1080229176  collection:"collection2"  file_count:23409  delete_count:5  deleted_byte_count:46268  replica_placement:100  version:3  modified_at_second:1614918481 
+          volume id:298  size:1075410024  collection:"collection2"  file_count:23222  delete_count:2  deleted_byte_count:46110  replica_placement:100  version:3  modified_at_second:1614918474 
+          volume id:302  size:1077559640  collection:"collection2"  file_count:23124  delete_count:7  deleted_byte_count:293111  replica_placement:100  version:3  modified_at_second:1614925500 
+          volume id:304  size:1081038944  collection:"collection2"  file_count:24505  delete_count:2  deleted_byte_count:124447  replica_placement:100  version:3  modified_at_second:1614925569 
+          volume id:305  size:1074185376  collection:"collection2"  file_count:22074  delete_count:5  deleted_byte_count:20221  replica_placement:100  version:3  modified_at_second:1614925312 
+          volume id:306  size:1074763952  collection:"collection2"  file_count:22939  replica_placement:100  version:3  modified_at_second:1614925462 
+          volume id:307  size:1076567912  collection:"collection2"  file_count:23377  delete_count:2  deleted_byte_count:25453  replica_placement:100  version:3  modified_at_second:1614931448 
+          volume id:308  size:1074022336  collection:"collection2"  file_count:23086  delete_count:2  deleted_byte_count:2127  replica_placement:100  version:3  modified_at_second:1614931401 
+          volume id:311  size:1088248344  collection:"collection2"  file_count:23553  delete_count:6  deleted_byte_count:191716  replica_placement:100  version:3  modified_at_second:1614931463 
+          volume id:312  size:1075037528  collection:"collection2"  file_count:22524  replica_placement:100  version:3  modified_at_second:1614937831 
+          volume id:313  size:1074875960  collection:"collection2"  file_count:22404  delete_count:4  deleted_byte_count:51728  replica_placement:100  version:3  modified_at_second:1614937755 
+          volume id:316  size:1077720776  collection:"collection2"  file_count:22605  delete_count:1  deleted_byte_count:8503  replica_placement:100  version:3  modified_at_second:1614937838 
+          volume id:318  size:1075965168  collection:"collection2"  file_count:22459  delete_count:2  deleted_byte_count:37778  replica_placement:100  version:3  modified_at_second:1614943862 
+          volume id:322  size:1078471536  collection:"collection2"  file_count:21905  delete_count:3  deleted_byte_count:145002  replica_placement:100  version:3  modified_at_second:1614950572 
+          volume id:323  size:1074608056  collection:"collection2"  file_count:21605  delete_count:4  deleted_byte_count:172090  replica_placement:100  version:3  modified_at_second:1614950526 
+          volume id:325  size:1080701232  collection:"collection2"  file_count:21735  replica_placement:100  version:3  modified_at_second:1614950525 
+          volume id:326  size:1076059920  collection:"collection2"  file_count:22564  delete_count:2  deleted_byte_count:192886  replica_placement:100  version:3  modified_at_second:1614950619 
+          volume id:327  size:1076121304  collection:"collection2"  file_count:22007  delete_count:3  deleted_byte_count:60358  replica_placement:100  version:3  modified_at_second:1614956487 
+          volume id:328  size:1074767816  collection:"collection2"  file_count:21720  delete_count:3  deleted_byte_count:56429  replica_placement:100  version:3  modified_at_second:1614956362 
+          volume id:329  size:1076691960  collection:"collection2"  file_count:22411  delete_count:5  deleted_byte_count:214092  replica_placement:100  version:3  modified_at_second:1614956485 
+          volume id:330  size:1080825760  collection:"collection2"  file_count:22464  delete_count:2  deleted_byte_count:15771  replica_placement:100  version:3  modified_at_second:1614956476 
+          volume id:331  size:1074957256  collection:"collection2"  file_count:21230  delete_count:4  deleted_byte_count:62145  replica_placement:100  version:3  modified_at_second:1614956259 
+          volume id:332  size:1075569928  collection:"collection2"  file_count:22097  delete_count:3  deleted_byte_count:98273  replica_placement:100  version:3  modified_at_second:1614962869 
+          volume id:333  size:1074270160  collection:"collection2"  file_count:21271  delete_count:2  deleted_byte_count:168122  replica_placement:100  version:3  modified_at_second:1614962697 
+          volume id:334  size:1075607880  collection:"collection2"  file_count:22546  delete_count:6  deleted_byte_count:101538  replica_placement:100  version:3  modified_at_second:1614962978 
+          volume id:335  size:1076235136  collection:"collection2"  file_count:22391  delete_count:3  deleted_byte_count:8838  replica_placement:100  version:3  modified_at_second:1614962970 
+          volume id:337  size:1075646896  collection:"collection2"  file_count:21934  delete_count:1  deleted_byte_count:3397  replica_placement:100  version:3  modified_at_second:1614969937 
+          volume id:339  size:1078402392  collection:"collection2"  file_count:22309  replica_placement:100  version:3  modified_at_second:1614969995 
+          volume id:340  size:1079462152  collection:"collection2"  file_count:22319  delete_count:4  deleted_byte_count:93620  replica_placement:100  version:3  modified_at_second:1614969977 
+          volume id:341  size:1074448360  collection:"collection2"  file_count:21590  delete_count:5  deleted_byte_count:160085  replica_placement:100  version:3  modified_at_second:1614969858 
+          volume id:343  size:1075345072  collection:"collection2"  file_count:21095  delete_count:2  deleted_byte_count:20581  replica_placement:100  version:3  modified_at_second:1614977148 
+          volume id:346  size:1076464112  collection:"collection2"  file_count:22320  delete_count:4  deleted_byte_count:798258  replica_placement:100  version:3  modified_at_second:1614977511 
+          volume id:347  size:1075145248  collection:"collection2"  file_count:22178  delete_count:1  deleted_byte_count:79392  replica_placement:100  version:3  modified_at_second:1614984727 
+          volume id:348  size:1080623544  collection:"collection2"  file_count:21667  delete_count:1  deleted_byte_count:2443  replica_placement:100  version:3  modified_at_second:1614984604 
+          volume id:349  size:1075957672  collection:"collection2"  file_count:22395  delete_count:2  deleted_byte_count:61565  replica_placement:100  version:3  modified_at_second:1614984748 
+          volume id:351  size:1078795120  collection:"collection2"  file_count:23660  delete_count:3  deleted_byte_count:102141  replica_placement:100  version:3  modified_at_second:1614984816 
+          volume id:352  size:1077145936  collection:"collection2"  file_count:22066  delete_count:1  deleted_byte_count:1018  replica_placement:100  version:3  modified_at_second:1614992130 
+          volume id:353  size:1074897496  collection:"collection2"  file_count:21266  delete_count:2  deleted_byte_count:3105374  replica_placement:100  version:3  modified_at_second:1614991951 
+          volume id:354  size:1085214104  collection:"collection2"  file_count:23150  delete_count:4  deleted_byte_count:82391  replica_placement:100  version:3  modified_at_second:1614992208 
+          volume id:357  size:1074276152  collection:"collection2"  file_count:23137  delete_count:4  deleted_byte_count:188487  replica_placement:100  version:3  modified_at_second:1614998792 
+          volume id:359  size:1074211296  collection:"collection2"  file_count:22437  delete_count:2  deleted_byte_count:187953  replica_placement:100  version:3  modified_at_second:1614998711 
+          volume id:360  size:1075532512  collection:"collection2"  file_count:22574  delete_count:3  deleted_byte_count:1774776  replica_placement:100  version:3  modified_at_second:1614998770 
+          volume id:361  size:1075362744  collection:"collection2"  file_count:22272  delete_count:1  deleted_byte_count:3497  replica_placement:100  version:3  modified_at_second:1614998668 
+          volume id:362  size:1074074176  collection:"collection2"  file_count:20595  delete_count:1  deleted_byte_count:112145  replica_placement:100  version:3  modified_at_second:1615004407 
+          volume id:363  size:1078859640  collection:"collection2"  file_count:23177  delete_count:4  deleted_byte_count:9601  replica_placement:100  version:3  modified_at_second:1615004823 
+          volume id:364  size:1081280880  collection:"collection2"  file_count:22686  delete_count:1  deleted_byte_count:84375  replica_placement:100  version:3  modified_at_second:1615004813 
+          volume id:365  size:1075736632  collection:"collection2"  file_count:22193  delete_count:5  deleted_byte_count:259033  replica_placement:100  version:3  modified_at_second:1615004776 
+          volume id:366  size:1075267272  collection:"collection2"  file_count:21856  delete_count:5  deleted_byte_count:138363  replica_placement:100  version:3  modified_at_second:1615004703 
+          volume id:367  size:1076403648  collection:"collection2"  file_count:22995  delete_count:2  deleted_byte_count:36955  replica_placement:100  version:3  modified_at_second:1615010985 
+          volume id:368  size:1074821960  collection:"collection2"  file_count:22252  delete_count:4  deleted_byte_count:3291946  replica_placement:100  version:3  modified_at_second:1615010877 
+          volume id:369  size:1091472040  collection:"collection2"  file_count:23709  delete_count:4  deleted_byte_count:400876  replica_placement:100  version:3  modified_at_second:1615011021 
+          volume id:370  size:1076040544  collection:"collection2"  file_count:22092  delete_count:2  deleted_byte_count:115388  replica_placement:100  version:3  modified_at_second:1615010877 
+          volume id:371  size:1078806216  collection:"collection2"  file_count:22685  delete_count:2  deleted_byte_count:68905  replica_placement:100  version:3  modified_at_second:1615010995 
+          volume id:372  size:1076193344  collection:"collection2"  file_count:22774  delete_count:1  deleted_byte_count:3495  replica_placement:100  version:3  modified_at_second:1615016911 
+          volume id:373  size:1080928088  collection:"collection2"  file_count:22617  delete_count:4  deleted_byte_count:91849  replica_placement:100  version:3  modified_at_second:1615016878 
+          volume id:374  size:1085011176  collection:"collection2"  file_count:23054  delete_count:2  deleted_byte_count:89034  replica_placement:100  version:3  modified_at_second:1615016917 
+          volume id:376  size:1074845832  collection:"collection2"  file_count:22908  delete_count:4  deleted_byte_count:432305  replica_placement:100  version:3  modified_at_second:1615016916 
+          volume id:377  size:957434264  collection:"collection2"  file_count:14929  delete_count:1  deleted_byte_count:43099  replica_placement:100  version:3  modified_at_second:1615632323 
+          volume id:379  size:1014108528  collection:"collection2"  file_count:15362  delete_count:6  deleted_byte_count:2481613  replica_placement:100  version:3  modified_at_second:1615632323 
+        Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
+      DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
+    Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
+  DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 
+total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 
+`

weed/shell/command_volume_server_evacuate_test.go

@@ -0,0 +1,17 @@
+package shell
+
+import (
+	"os"
+	"testing"
+)
+
+func TestVolumeServerEvacuate(t *testing.T) {
+	topologyInfo := parseOutput(topoData)
+
+	volumeServer := "192.168.1.4:8080"
+
+	if err := evacuateNormalVolumes(nil, topologyInfo, volumeServer, true, false, os.Stdout); err != nil {
+		t.Errorf("evacuate: %v", err)
+	}
+
+}

weed/shell/shell_liner.go

@@ -19,6 +19,10 @@ var (
 
 func RunShell(options ShellOptions) {
 
+	sort.Slice(Commands, func(i, j int) bool {
+		return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
+	})
+
 	line = liner.NewLiner()
 	defer line.Close()
 
@@ -96,9 +100,6 @@ func printGenericHelp() {
 `
 	fmt.Print(msg)
 
-	sort.Slice(Commands, func(i, j int) bool {
-		return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
-	})
 	for _, c := range Commands {
 		helpTexts := strings.SplitN(c.Help(), "\n", 2)
 		fmt.Printf("  %-30s\t# %s \n", c.Name(), helpTexts[0])
@@ -114,10 +115,6 @@ func printHelp(cmds []string) {
 	} else {
 		cmd := strings.ToLower(args[0])
 
-		sort.Slice(Commands, func(i, j int) bool {
-			return strings.Compare(Commands[i].Name(), Commands[j].Name()) < 0
-		})
-
 		for _, c := range Commands {
 			if c.Name() == cmd {
 				fmt.Printf("  %s\t# %s\n", c.Name(), c.Help())

weed/storage/needle/crc.go

@@ -35,7 +35,7 @@ func NewCRCwriter(w io.Writer) *CRCwriter {
 
 	return &CRCwriter{
 		crc: CRC(0),
-		w: w,
+		w:   w,
 	}
 
 }

weed/storage/needle/needle_read_write.go

@@ -3,13 +3,12 @@ package needle
 import (
 	"errors"
 	"fmt"
-	"io"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
+	"math"
 )
 
 const (
@@ -156,6 +155,35 @@ func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset u
 	return offset, size, actualSize, err
 }
 
+func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) {
+
+	if end, _, e := w.GetStat(); e == nil {
+		defer func(w backend.BackendStorageFile, off int64) {
+			if err != nil {
+				if te := w.Truncate(end); te != nil {
+					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
+				}
+			}
+		}(w, end)
+		offset = uint64(end)
+	} else {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
+		return
+	}
+
+	if version == Version3 {
+		tsOffset := NeedleHeaderSize + size + NeedleChecksumSize
+		util.Uint64toBytes(dataSlice[tsOffset:tsOffset+TimestampSize], appendAtNs)
+	}
+
+	if err == nil {
+		_, err = w.WriteAt(dataSlice, int64(offset))
+	}
+
+	return
+
+}
+
 func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) {
 
 	dataSize := GetActualSize(size, version)

weed/storage/needle_map/memdb.go

@@ -2,6 +2,7 @@ package needle_map
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -104,7 +105,13 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
 	}
 	defer idxFile.Close()
 
-	return idx.WalkIndexFile(idxFile, func(key NeedleId, offset Offset, size Size) error {
+	return cm.LoadFromReaderAt(idxFile)
+
+}
+
+func (cm *MemDb) LoadFromReaderAt(readerAt io.ReaderAt) (ret error) {
+
+	return idx.WalkIndexFile(readerAt, func(key NeedleId, offset Offset, size Size) error {
 		if offset.IsZero() || size.IsDeleted() {
 			return cm.Delete(key)
 		}

weed/storage/store.go

@@ -217,6 +217,9 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 		location.volumesLock.RLock()
 		for _, v := range location.volumes {
 			curMaxFileKey, volumeMessage := v.ToVolumeInformationMessage()
+			if volumeMessage == nil {
+				continue
+			}
 			if maxFileKey < curMaxFileKey {
 				maxFileKey = curMaxFileKey
 			}