mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'master' of https://github.com/chrislusf/seaweedfs into chrislusf-master
This commit is contained in:
guosj
commit
82ee31965d
9
.github/workflows/binaries_dev.yml
vendored
9
.github/workflows/binaries_dev.yml
vendored
|
|
@ -4,9 +4,14 @@ on:
|
|||
push:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
cleanup:
|
||||
permissions:
|
||||
contents: write # for mknejp/delete-release-assets to delete release assets
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
|
@ -21,6 +26,8 @@ jobs:
|
|||
weed-*
|
||||
|
||||
build_dev_linux_windows:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
|
|
@ -68,6 +75,8 @@ jobs:
|
|||
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
build_dev_darwin:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
needs: build_dev_linux_windows
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
|
|
|
|||
5
.github/workflows/binaries_release0.yml
vendored
5
.github/workflows/binaries_release0.yml
vendored
|
|
@ -11,9 +11,14 @@ on:
|
|||
workflow_dispatch:
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-release-binaries_windows:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
|
|
|
|||
5
.github/workflows/binaries_release1.yml
vendored
5
.github/workflows/binaries_release1.yml
vendored
|
|
@ -11,9 +11,14 @@ on:
|
|||
workflow_dispatch:
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-release-binaries_linux:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
|
|
|
|||
5
.github/workflows/binaries_release2.yml
vendored
5
.github/workflows/binaries_release2.yml
vendored
|
|
@ -11,9 +11,14 @@ on:
|
|||
workflow_dispatch:
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-release-binaries_darwin:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
|
|
|
|||
5
.github/workflows/binaries_release3.yml
vendored
5
.github/workflows/binaries_release3.yml
vendored
|
|
@ -11,9 +11,14 @@ on:
|
|||
workflow_dispatch:
|
||||
|
||||
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-release-binaries_freebsd:
|
||||
permissions:
|
||||
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
|
|
|
|||
2
.github/workflows/container_dev.yml
vendored
2
.github/workflows/container_dev.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_latest.yml
vendored
2
.github/workflows/container_latest.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release1.yml
vendored
2
.github/workflows/container_release1.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release2.yml
vendored
2
.github/workflows/container_release2.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release3.yml
vendored
2
.github/workflows/container_release3.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
|
||||
uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ Table of Contents
|
|||
* [Quick Start](#quick-start)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
|
|
@ -82,6 +83,9 @@ Table of Contents
|
|||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
## Quick Start SeaweedFS S3 on AWS ##
|
||||
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
|
||||
|
||||
## Introduction ##
|
||||
|
||||
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM golang:1.17-alpine as builder
|
||||
FROM golang:1.18-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM golang:1.17-alpine as builder
|
||||
FROM golang:1.18-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
FROM golang:1.17-buster as builder
|
||||
FROM golang:1.18-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v6.22.1
|
||||
ENV ROCKSDB_VERSION v7.0.4
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
|
|
|
|||
|
|
@ -55,6 +55,9 @@ cluster: build
|
|||
2clusters: build
|
||||
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
|
||||
|
||||
hashicorp_raft: build
|
||||
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
|
||||
s3tests: build s3tests_build
|
||||
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
|
||||
|
||||
|
|
|
|||
89
docker/compose/local-hashicorp-raft-compose.yml
Normal file
89
docker/compose/local-hashicorp-raft-compose.yml
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
version: '2'
|
||||
|
||||
services:
|
||||
master0:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/0:/data
|
||||
environment:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
|
||||
master1:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 9334:9334
|
||||
- 19334:19334
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/1:/data
|
||||
environment:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
|
||||
master2:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 9335:9335
|
||||
- 19335:19335
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/2:/data
|
||||
environment:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
|
||||
volume1:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8080:8080
|
||||
- 18080:18080
|
||||
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
volume2:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8082:8082
|
||||
- 18082:18082
|
||||
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
volume3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8083:8083
|
||||
- 18083:18083
|
||||
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
filer:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 8111:8111
|
||||
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
- volume1
|
||||
- volume2
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- 8333:8333
|
||||
command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"'
|
||||
depends_on:
|
||||
- master0
|
||||
- master1
|
||||
- volume1
|
||||
- volume2
|
||||
- filer
|
||||
|
|
@ -24,7 +24,7 @@ services:
|
|||
- 8888:8888
|
||||
- 18888:18888
|
||||
- 8000:8000
|
||||
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
|
||||
command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false'
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
depends_on:
|
||||
|
|
@ -38,7 +38,7 @@ services:
|
|||
S3TEST_CONF: "s3tests.conf"
|
||||
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
|
||||
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
|
||||
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
|
||||
NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)"
|
||||
depends_on:
|
||||
- master
|
||||
- volume
|
||||
|
|
|
|||
153
go.mod
153
go.mod
|
|
@ -4,13 +4,13 @@ go 1.18
|
|||
|
||||
require (
|
||||
cloud.google.com/go v0.100.2 // indirect
|
||||
cloud.google.com/go/pubsub v1.19.0
|
||||
cloud.google.com/go/storage v1.21.0
|
||||
cloud.google.com/go/pubsub v1.20.0
|
||||
cloud.google.com/go/storage v1.22.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.14.0
|
||||
github.com/OneOfOne/xxhash v1.2.2
|
||||
github.com/Shopify/sarama v1.23.1
|
||||
github.com/aws/aws-sdk-go v1.43.25
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/Shopify/sarama v1.32.0
|
||||
github.com/aws/aws-sdk-go v1.43.41
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
|
|
@ -33,11 +33,10 @@ require (
|
|||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.17.0
|
||||
github.com/frankban/quicktest v1.7.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.4.1
|
||||
github.com/go-redsync/redsync/v4 v4.5.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/go-zookeeper/zk v1.0.2 // indirect
|
||||
|
|
@ -50,9 +49,9 @@ require (
|
|||
github.com/google/go-cmp v0.5.7 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
|
||||
github.com/googleapis/gax-go/v2 v2.3.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.0 // indirect
|
||||
|
|
@ -65,41 +64,40 @@ require (
|
|||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
github.com/klauspost/cpuid v1.2.1 // indirect
|
||||
github.com/klauspost/reedsolomon v1.9.2
|
||||
github.com/klauspost/compress v1.15.1 // indirect
|
||||
github.com/klauspost/reedsolomon v1.9.16
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.4
|
||||
github.com/linxGnu/grocksdb v1.6.38
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mailru/easyjson v0.7.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||
github.com/lib/pq v1.10.5
|
||||
github.com/linxGnu/grocksdb v1.7.0
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.3 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.19
|
||||
github.com/pelletier/go-toml v1.7.0 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.32
|
||||
github.com/pelletier/go-toml v1.9.4 // indirect
|
||||
github.com/peterh/liner v1.2.2
|
||||
github.com/pierrec/lz4 v2.2.7+incompatible // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
github.com/prometheus/client_golang v1.12.1
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.26.0 // indirect
|
||||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.7.0 // indirect
|
||||
github.com/spf13/cast v1.3.0 // indirect
|
||||
github.com/spf13/afero v1.8.2 // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.4.0
|
||||
github.com/spf13/viper v1.11.0
|
||||
github.com/streadway/amqp v1.0.0
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
|
|
@ -114,88 +112,103 @@ require (
|
|||
github.com/viant/ptrie v0.3.0
|
||||
github.com/viant/toolbox v0.33.2 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.0.2 // indirect
|
||||
github.com/xdg-go/scram v1.1.0 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.2 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.0
|
||||
go.mongodb.org/mongo-driver v1.8.4
|
||||
go.etcd.io/etcd/client/v3 v3.5.3
|
||||
go.mongodb.org/mongo-driver v1.9.0
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
gocloud.dev v0.24.0
|
||||
gocloud.dev/pubsub/natspubsub v0.20.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.24.0
|
||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
|
||||
gocloud.dev v0.25.0
|
||||
gocloud.dev/pubsub/natspubsub v0.25.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.25.0
|
||||
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
|
||||
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
|
||||
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
|
||||
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
|
||||
golang.org/x/net v0.0.0-20220412020605-290c469a71a5
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/api v0.73.0
|
||||
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
|
||||
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
|
||||
google.golang.org/api v0.74.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
|
||||
google.golang.org/grpc v1.45.0
|
||||
google.golang.org/protobuf v1.28.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.35.24 // indirect
|
||||
modernc.org/ccgo/v3 v3.15.17 // indirect
|
||||
modernc.org/ccgo/v3 v3.15.18 // indirect
|
||||
modernc.org/libc v1.14.12 // indirect
|
||||
modernc.org/mathutil v1.4.1 // indirect
|
||||
modernc.org/memory v1.0.7 // indirect
|
||||
modernc.org/opt v0.1.1 // indirect
|
||||
modernc.org/sqlite v1.15.3
|
||||
modernc.org/sqlite v1.16.0
|
||||
modernc.org/strutil v1.1.1 // indirect
|
||||
modernc.org/token v1.0.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/fluent/fluent-logger-golang v1.8.0
|
||||
github.com/Jille/raft-grpc-transport v1.2.0
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/hanwen/go-fuse/v2 v2.1.0
|
||||
github.com/hashicorp/raft v1.3.7
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.5.0 // indirect
|
||||
cloud.google.com/go/iam v0.1.1 // indirect
|
||||
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
|
||||
github.com/aws/smithy-go v1.8.0 // indirect
|
||||
cloud.google.com/go/iam v0.3.0 // indirect
|
||||
github.com/armon/go-metrics v0.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
|
||||
github.com/aws/smithy-go v1.11.2 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 // indirect
|
||||
github.com/googleapis/go-type-adapters v1.0.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.7 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/nats-io/nats.go v1.11.0 // indirect
|
||||
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.3 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.3 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.7.0 // indirect
|
||||
go.uber.org/zap v1.19.0 // indirect
|
||||
golang.org/x/mod v0.5.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
gopkg.in/ini.v1 v1.66.4 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
lukechampine.com/uint128 v1.1.1 // indirect
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "2.95"
|
||||
version: "2.95"
|
||||
appVersion: "2.99"
|
||||
version: "2.99"
|
||||
|
|
|
|||
|
|
@ -154,13 +154,16 @@ spec:
|
|||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
{{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}
|
||||
volumeMounts:
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
- mountPath: /etc/sw
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- if .Values.filer.enablePVC }}
|
||||
- name: data-filer
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
|
|
@ -183,7 +186,6 @@ spec:
|
|||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.filer.port }}
|
||||
name: swfs-filer
|
||||
|
|
@ -250,16 +252,18 @@ spec:
|
|||
nodeSelector:
|
||||
{{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{/* volumeClaimTemplates:*/}}
|
||||
{{/* - metadata:*/}}
|
||||
{{/* name: data-{{ .Release.Namespace }}*/}}
|
||||
{{/* spec:*/}}
|
||||
{{/* accessModes:*/}}
|
||||
{{/* - ReadWriteOnce*/}}
|
||||
{{/* resources:*/}}
|
||||
{{/* requests:*/}}
|
||||
{{/* storage: {{ .Values.filer.storage }}*/}}
|
||||
{{/* {{- if .Values.filer.storageClass }}*/}}
|
||||
{{/* storageClassName: {{ .Values.filer.storageClass }}*/}}
|
||||
{{/* {{- end }}*/}}
|
||||
{{- if .Values.filer.enablePVC }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data-filer
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.filer.storage }}
|
||||
{{- if .Values.filer.storageClass }}
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ master:
|
|||
grpcPort: 19333
|
||||
ipBind: "0.0.0.0"
|
||||
volumePreallocate: false
|
||||
#Master stops directing writes to oversized volumes
|
||||
volumeSizeLimitMB: 30000
|
||||
volumeSizeLimitMB: 1000
|
||||
loggingOverrideLevel: null
|
||||
#number of seconds between heartbeats, default 5
|
||||
pulseSeconds: null
|
||||
|
|
@ -62,6 +61,8 @@ master:
|
|||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
# enablePVC will create a pvc for filer for data persistence.
|
||||
enablePVC: false
|
||||
# storage and storageClass are the settings for configuring stateful
|
||||
# storage for the master pods. storage should be set to the disk size of
|
||||
# the attached volume. storageClass is the class of storage which defaults
|
||||
|
|
@ -358,6 +359,7 @@ filer:
|
|||
WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600"
|
||||
# enable usage of memsql as filer backend
|
||||
WEED_MYSQL_INTERPOLATEPARAMS: "true"
|
||||
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
|
|
|
|||
|
|
@ -48,6 +48,9 @@ service SeaweedFiler {
|
|||
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
|
||||
}
|
||||
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
|
||||
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
|
||||
}
|
||||
|
||||
|
|
@ -311,6 +314,16 @@ message StatisticsResponse {
|
|||
uint64 file_count = 6;
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
|
||||
message GetFilerConfigurationRequest {
|
||||
}
|
||||
message GetFilerConfigurationResponse {
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.2</version>
|
||||
<version>3.2.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
<properties>
|
||||
<seaweedfs.client.version>2.85</seaweedfs.client.version>
|
||||
<hadoop.version>3.1.4</hadoop.version>
|
||||
<hadoop.version>3.2.3</hadoop.version>
|
||||
</properties>
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
MasterType = "master"
|
||||
FilerType = "filer"
|
||||
BrokerType = "broker"
|
||||
MasterType = "master"
|
||||
VolumeServerType = "volumeServer"
|
||||
FilerType = "filer"
|
||||
BrokerType = "broker"
|
||||
)
|
||||
|
||||
type ClusterNode struct {
|
||||
|
|
@ -80,6 +81,15 @@ func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress
|
|||
},
|
||||
}
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -119,6 +129,15 @@ func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddr
|
|||
}
|
||||
}
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
|
|||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
go b.masterClient.KeepConnectedToMaster()
|
||||
b.masterClient.WaitUntilConnected()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
|
@ -29,7 +30,7 @@ var (
|
|||
)
|
||||
|
||||
type FilerOptions struct {
|
||||
masters []pb.ServerAddress
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
|
|
@ -89,6 +90,7 @@ func init() {
|
|||
filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
|
||||
filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
|
||||
filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
|
||||
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
|
||||
// start webdav on filer
|
||||
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
|
||||
|
|
@ -171,7 +173,7 @@ func runFiler(cmd *Command, args []string) bool {
|
|||
}()
|
||||
}
|
||||
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
|
||||
|
||||
f.startFiler()
|
||||
|
||||
|
|
@ -247,18 +249,6 @@ func (fo *FilerOptions) startFiler() {
|
|||
glog.Fatalf("Filer listener error: %v", e)
|
||||
}
|
||||
|
||||
// start on local unix socket
|
||||
if *fo.localSocket == "" {
|
||||
*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
|
||||
if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
|
||||
}
|
||||
}
|
||||
filerSocketListener, err := net.Listen("unix", *fo.localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
|
||||
}
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *fo.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
|
||||
|
|
@ -274,9 +264,22 @@ func (fo *FilerOptions) startFiler() {
|
|||
go grpcS.Serve(grpcL)
|
||||
|
||||
httpS := &http.Server{Handler: defaultMux}
|
||||
go func() {
|
||||
httpS.Serve(filerSocketListener)
|
||||
}()
|
||||
if runtime.GOOS != "windows" {
|
||||
if *fo.localSocket == "" {
|
||||
*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
|
||||
if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
// start on local unix socket
|
||||
filerSocketListener, err := net.Listen("unix", *fo.localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
|
||||
}
|
||||
httpS.Serve(filerSocketListener)
|
||||
}()
|
||||
}
|
||||
if filerLocalListener != nil {
|
||||
go func() {
|
||||
if err := httpS.Serve(filerLocalListener); err != nil {
|
||||
|
|
|
|||
|
|
@ -267,7 +267,10 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
|
|||
return nil
|
||||
}
|
||||
key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
|
||||
return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
|
||||
if !dataSink.IsIncremental() {
|
||||
return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handle new entries
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
|
|||
}
|
||||
}
|
||||
|
||||
masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
|
||||
masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
|
||||
Masters: masters,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"golang.org/x/exp/slices"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -48,6 +48,8 @@ type MasterOptions struct {
|
|||
metricsHttpPort *int
|
||||
heartbeatInterval *time.Duration
|
||||
electionTimeout *time.Duration
|
||||
raftHashicorp *bool
|
||||
raftBootstrap *bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -71,6 +73,8 @@ func init() {
|
|||
m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
|
||||
m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
|
||||
m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
}
|
||||
|
||||
var cmdMaster = &Command{
|
||||
|
|
@ -132,8 +136,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
|||
|
||||
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
|
||||
|
||||
masterPeers := make(map[string]pb.ServerAddress)
|
||||
for _, peer := range peers {
|
||||
masterPeers[string(peer)] = peer
|
||||
}
|
||||
|
||||
r := mux.NewRouter()
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
|
||||
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
|
||||
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
|
||||
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
|
||||
masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
|
||||
|
|
@ -144,20 +153,32 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
|||
// start raftServer
|
||||
raftServerOption := &weed_server.RaftServerOption{
|
||||
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
|
||||
Peers: peers,
|
||||
Peers: masterPeers,
|
||||
ServerAddr: myMasterAddress,
|
||||
DataDir: util.ResolvePath(*masterOption.metaFolder),
|
||||
Topo: ms.Topo,
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
HeartbeatInterval: *masterOption.heartbeatInterval,
|
||||
ElectionTimeout: *masterOption.electionTimeout,
|
||||
RaftBootstrap: *m.raftBootstrap,
|
||||
}
|
||||
raftServer, err := weed_server.NewRaftServer(raftServerOption)
|
||||
if raftServer == nil {
|
||||
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
|
||||
var raftServer *weed_server.RaftServer
|
||||
var err error
|
||||
if *m.raftHashicorp {
|
||||
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
|
||||
glog.Fatalf("NewHashicorpRaftServer: %s", err)
|
||||
}
|
||||
} else {
|
||||
raftServer, err = weed_server.NewRaftServer(raftServerOption)
|
||||
if raftServer == nil {
|
||||
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
|
||||
}
|
||||
}
|
||||
ms.SetRaftServer(raftServer)
|
||||
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
||||
if *m.raftHashicorp {
|
||||
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
|
||||
}
|
||||
// starting grpc server
|
||||
grpcPort := *masterOption.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
|
||||
|
|
@ -166,7 +187,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
|||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
|
||||
master_pb.RegisterSeaweedServer(grpcS, ms)
|
||||
protobuf.RegisterRaftServer(grpcS, raftServer)
|
||||
if *m.raftHashicorp {
|
||||
raftServer.TransportManager.Register(grpcS)
|
||||
} else {
|
||||
protobuf.RegisterRaftServer(grpcS, raftServer)
|
||||
}
|
||||
reflection.Register(grpcS)
|
||||
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
|
||||
if grpcLocalL != nil {
|
||||
|
|
@ -174,14 +199,17 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
|||
}
|
||||
go grpcS.Serve(grpcL)
|
||||
|
||||
go func() {
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
|
||||
if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
|
||||
raftServer.DoJoinCommand()
|
||||
timeSleep := 1500 * time.Millisecond
|
||||
if !*m.raftHashicorp {
|
||||
go func() {
|
||||
time.Sleep(timeSleep)
|
||||
if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
|
||||
if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
|
||||
raftServer.DoJoinCommand()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}()
|
||||
}
|
||||
|
||||
go ms.MasterClient.KeepConnectedToMaster()
|
||||
|
||||
|
|
@ -246,8 +274,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
|
|||
}
|
||||
|
||||
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
|
||||
sort.Slice(peers, func(i, j int) bool {
|
||||
return strings.Compare(string(peers[i]), string(peers[j])) < 0
|
||||
slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
|
||||
return strings.Compare(string(a), string(b)) < 0
|
||||
})
|
||||
if len(peers) <= 0 {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
|
|||
func startMasterFollower(masterOptions MasterOptions) {
|
||||
|
||||
// collect settings from main masters
|
||||
masters := pb.ServerAddresses(*mf.peers).ToAddresses()
|
||||
masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
|
||||
|
||||
var err error
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ type MountOptions struct {
|
|||
readOnly *bool
|
||||
debug *bool
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
@ -63,6 +64,7 @@ func init() {
|
|||
mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
|
||||
mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
|
||||
mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
|
||||
mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")
|
||||
|
||||
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
|
||||
|
|
|
|||
|
|
@ -12,9 +12,12 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/mount/unmount"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
|
|
@ -98,6 +101,22 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
|||
|
||||
unmount.Unmount(dir)
|
||||
|
||||
// start on local unix socket
|
||||
if *option.localSocket == "" {
|
||||
mountDirHash := util.HashToInt32([]byte(dir))
|
||||
if mountDirHash < 0 {
|
||||
mountDirHash = -mountDirHash
|
||||
}
|
||||
*option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash)
|
||||
}
|
||||
if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
|
||||
}
|
||||
montSocketListener, err := net.Listen("unix", *option.localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
|
||||
}
|
||||
|
||||
// detect mount folder mode
|
||||
if *option.dirAutoCreate {
|
||||
os.MkdirAll(dir, os.FileMode(0777)&^umask)
|
||||
|
|
@ -229,6 +248,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
|||
unmount.Unmount(dir)
|
||||
})
|
||||
|
||||
grpcS := pb.NewGrpcServer()
|
||||
mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
|
||||
reflection.Register(grpcS)
|
||||
go grpcS.Serve(montSocketListener)
|
||||
|
||||
seaweedFileSystem.StartBackgroundTasks()
|
||||
|
||||
fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
|
|
|
|||
|
|
@ -24,17 +24,18 @@ var (
|
|||
)
|
||||
|
||||
type S3Options struct {
|
||||
filer *string
|
||||
bindIp *string
|
||||
port *int
|
||||
config *string
|
||||
domainName *string
|
||||
tlsPrivateKey *string
|
||||
tlsCertificate *string
|
||||
metricsHttpPort *int
|
||||
allowEmptyFolder *bool
|
||||
auditLogConfig *string
|
||||
localFilerSocket *string
|
||||
filer *string
|
||||
bindIp *string
|
||||
port *int
|
||||
config *string
|
||||
domainName *string
|
||||
tlsPrivateKey *string
|
||||
tlsCertificate *string
|
||||
metricsHttpPort *int
|
||||
allowEmptyFolder *bool
|
||||
allowDeleteBucketNotEmpty *bool
|
||||
auditLogConfig *string
|
||||
localFilerSocket *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -49,6 +50,7 @@ func init() {
|
|||
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
|
||||
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
|
||||
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
}
|
||||
|
||||
var cmdS3 = &Command{
|
||||
|
|
@ -178,14 +180,15 @@ func (s3opt *S3Options) startS3Server() bool {
|
|||
router := mux.NewRouter().SkipClean(true)
|
||||
|
||||
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
|
||||
Filer: filerAddress,
|
||||
Port: *s3opt.port,
|
||||
Config: *s3opt.config,
|
||||
DomainName: *s3opt.domainName,
|
||||
BucketsPath: filerBucketsPath,
|
||||
GrpcDialOption: grpcDialOption,
|
||||
AllowEmptyFolder: *s3opt.allowEmptyFolder,
|
||||
LocalFilerSocket: s3opt.localFilerSocket,
|
||||
Filer: filerAddress,
|
||||
Port: *s3opt.port,
|
||||
Config: *s3opt.config,
|
||||
DomainName: *s3opt.domainName,
|
||||
BucketsPath: filerBucketsPath,
|
||||
GrpcDialOption: grpcDialOption,
|
||||
AllowEmptyFolder: *s3opt.allowEmptyFolder,
|
||||
AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
|
||||
LocalFilerSocket: s3opt.localFilerSocket,
|
||||
})
|
||||
if s3ApiServer_err != nil {
|
||||
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
|
||||
|
|
|
|||
|
|
@ -138,6 +138,7 @@ func init() {
|
|||
s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
|
||||
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
|
||||
s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
|
||||
s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
|
||||
iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
|
||||
|
||||
|
|
@ -191,7 +192,7 @@ func runServer(cmd *Command, args []string) bool {
|
|||
// ip address
|
||||
masterOptions.ip = serverIp
|
||||
masterOptions.ipBind = serverBindIp
|
||||
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
|
||||
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
|
||||
filerOptions.ip = serverIp
|
||||
filerOptions.bindIp = serverBindIp
|
||||
s3Options.bindIp = serverBindIp
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import (
|
|||
"bytes"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/exp/slices"
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
|
|
@ -23,6 +23,9 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
|
|||
}
|
||||
|
||||
func FileSize(entry *filer_pb.Entry) (size uint64) {
|
||||
if entry == nil || entry.Attributes == nil {
|
||||
return 0
|
||||
}
|
||||
fileSize := entry.Attributes.FileSize
|
||||
if entry.RemoteEntry != nil {
|
||||
if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
|
||||
|
|
@ -251,19 +254,17 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
|
|||
if true {
|
||||
return visibles2, err
|
||||
}
|
||||
|
||||
sort.Slice(chunks, func(i, j int) bool {
|
||||
if chunks[i].Mtime == chunks[j].Mtime {
|
||||
filer_pb.EnsureFid(chunks[i])
|
||||
filer_pb.EnsureFid(chunks[j])
|
||||
if chunks[i].Fid == nil || chunks[j].Fid == nil {
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
|
||||
if a.Mtime == b.Mtime {
|
||||
filer_pb.EnsureFid(a)
|
||||
filer_pb.EnsureFid(b)
|
||||
if a.Fid == nil || b.Fid == nil {
|
||||
return true
|
||||
}
|
||||
return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
|
||||
return a.Fid.FileKey < b.Fid.FileKey
|
||||
}
|
||||
return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run
|
||||
return a.Mtime < b.Mtime
|
||||
})
|
||||
|
||||
for _, chunk := range chunks {
|
||||
|
||||
// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"golang.org/x/exp/slices"
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
|
@ -34,11 +34,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
|
|||
}
|
||||
|
||||
func printChunks(name string, chunks []*filer_pb.FileChunk) {
|
||||
sort.Slice(chunks, func(i, j int) bool {
|
||||
if chunks[i].Offset == chunks[j].Offset {
|
||||
return chunks[i].Mtime < chunks[j].Mtime
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
|
||||
if a.Offset == b.Offset {
|
||||
return a.Mtime < b.Mtime
|
||||
}
|
||||
return chunks[i].Offset < chunks[j].Offset
|
||||
return a.Offset < b.Offset
|
||||
})
|
||||
for _, chunk := range chunks {
|
||||
glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package filer
|
|||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"sort"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
|
||||
|
|
@ -22,17 +22,14 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
|
|||
isStart: false,
|
||||
})
|
||||
}
|
||||
sort.Slice(points, func(i, j int) bool {
|
||||
if points[i].x != points[j].x {
|
||||
return points[i].x < points[j].x
|
||||
slices.SortFunc(points, func(a, b *Point) bool {
|
||||
if a.x != b.x {
|
||||
return a.x < b.x
|
||||
}
|
||||
if points[i].ts != points[j].ts {
|
||||
return points[i].ts < points[j].ts
|
||||
if a.ts != b.ts {
|
||||
return a.ts < b.ts
|
||||
}
|
||||
if !points[i].isStart {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
return !a.isStart
|
||||
})
|
||||
|
||||
var prevX int64
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ type Filer struct {
|
|||
UniqueFileId uint32
|
||||
}
|
||||
|
||||
func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
|
||||
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption,
|
||||
filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
|
||||
f := &Filer{
|
||||
MasterClient: wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),
|
||||
|
|
|
|||
|
|
@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
|
|||
}
|
||||
} else {
|
||||
if _, found := ma.peerStatues[address]; found {
|
||||
ma.peerStatues[address] -= 1
|
||||
}
|
||||
if ma.peerStatues[address] <= 0 {
|
||||
delete(ma.peerStatues, address)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package redis
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"golang.org/x/exp/slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -157,8 +157,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
|
|||
}
|
||||
|
||||
// sort
|
||||
sort.Slice(members, func(i, j int) bool {
|
||||
return strings.Compare(members[i], members[j]) < 0
|
||||
slices.SortFunc(members, func(a, b string) bool {
|
||||
return strings.Compare(a, b) < 0
|
||||
})
|
||||
|
||||
// limit
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package filer
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"golang.org/x/exp/slices"
|
||||
"io"
|
||||
"math"
|
||||
"sort"
|
||||
|
|
@ -39,11 +40,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool {
|
|||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
sort.Slice(a, func(i, j int) bool {
|
||||
return strings.Compare(a[i].ETag, a[j].ETag) < 0
|
||||
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
|
||||
return strings.Compare(i.ETag, j.ETag) < 0
|
||||
})
|
||||
sort.Slice(b, func(i, j int) bool {
|
||||
return strings.Compare(b[i].ETag, b[j].ETag) < 0
|
||||
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
|
||||
return strings.Compare(i.ETag, j.ETag) < 0
|
||||
})
|
||||
for i := 0; i < len(a); i++ {
|
||||
if a[i].ETag != b[i].ETag {
|
||||
|
|
@ -179,8 +180,8 @@ var _ = io.ReaderAt(&ChunkStreamReader{})
|
|||
func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
|
||||
|
||||
chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
|
||||
sort.Slice(chunkViews, func(i, j int) bool {
|
||||
return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset
|
||||
slices.SortFunc(chunkViews, func(a, b *ChunkView) bool {
|
||||
return a.LogicOffset < b.LogicOffset
|
||||
})
|
||||
|
||||
var totalSize int64
|
||||
|
|
|
|||
|
|
@ -4,10 +4,6 @@ import (
|
|||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -16,6 +12,11 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
)
|
||||
|
||||
|
|
@ -155,6 +156,22 @@ func (iama *IamApiServer) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName str
|
|||
return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
|
||||
}
|
||||
|
||||
func (iama *IamApiServer) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp UpdateUserResponse, err error) {
|
||||
userName := values.Get("UserName")
|
||||
newUserName := values.Get("NewUserName")
|
||||
if newUserName != "" {
|
||||
for _, ident := range s3cfg.Identities {
|
||||
if userName == ident.Name {
|
||||
ident.Name = newUserName
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return resp, nil
|
||||
}
|
||||
return resp, fmt.Errorf(iam.ErrCodeNoSuchEntityException)
|
||||
}
|
||||
|
||||
func GetPolicyDocument(policy *string) (policyDocument PolicyDocument, err error) {
|
||||
if err = json.Unmarshal([]byte(*policy), &policyDocument); err != nil {
|
||||
return PolicyDocument{}, err
|
||||
|
|
@ -396,6 +413,13 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
changed = false
|
||||
case "UpdateUser":
|
||||
response, err = iama.UpdateUser(s3cfg, values)
|
||||
if err != nil {
|
||||
glog.Errorf("UpdateUser: %+v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
|
||||
return
|
||||
}
|
||||
case "DeleteUser":
|
||||
userName := values.Get("UserName")
|
||||
response, err = iama.DeleteUser(s3cfg, userName)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,11 @@ type GetUserResponse struct {
|
|||
} `xml:"GetUserResult"`
|
||||
}
|
||||
|
||||
type UpdateUserResponse struct {
|
||||
CommonResponse
|
||||
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateUserResponse"`
|
||||
}
|
||||
|
||||
type CreateAccessKeyResponse struct {
|
||||
CommonResponse
|
||||
XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"`
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ type IamS3ApiConfigure struct {
|
|||
}
|
||||
|
||||
type IamServerOption struct {
|
||||
Masters []pb.ServerAddress
|
||||
Masters map[string]pb.ServerAddress
|
||||
Filer pb.ServerAddress
|
||||
Port int
|
||||
GrpcDialOption grpc.DialOption
|
||||
|
|
|
|||
|
|
@ -2,6 +2,10 @@ package iamapi
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
|
|
@ -9,9 +13,6 @@ import (
|
|||
"github.com/gorilla/mux"
|
||||
"github.com/jinzhu/copier"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var GetS3ApiConfiguration func(s3cfg *iam_pb.S3ApiConfiguration) (err error)
|
||||
|
|
@ -161,8 +162,20 @@ func TestGetUserPolicy(t *testing.T) {
|
|||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
}
|
||||
|
||||
func TestDeleteUser(t *testing.T) {
|
||||
func TestUpdateUser(t *testing.T) {
|
||||
userName := aws.String("Test")
|
||||
newUserName := aws.String("Test-New")
|
||||
params := &iam.UpdateUserInput{NewUserName: newUserName, UserName: userName}
|
||||
req, _ := iam.New(session.New()).UpdateUserRequest(params)
|
||||
_ = req.Build()
|
||||
out := UpdateUserResponse{}
|
||||
response, err := executeRequest(req.HTTPRequest, out)
|
||||
assert.Equal(t, nil, err)
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
}
|
||||
|
||||
func TestDeleteUser(t *testing.T) {
|
||||
userName := aws.String("Test-New")
|
||||
params := &iam.DeleteUserInput{UserName: userName}
|
||||
req, _ := iam.New(session.New()).DeleteUserRequest(params)
|
||||
_ = req.Build()
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"golang.org/x/exp/slices"
|
||||
"io"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
|
|
@ -76,8 +76,8 @@ func (fh *FileHandle) addChunks(chunks []*filer_pb.FileChunk) {
|
|||
}
|
||||
|
||||
// sort incoming chunks
|
||||
sort.Slice(chunks, func(i, j int) bool {
|
||||
return lessThan(chunks[i], chunks[j])
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
|
||||
return lessThan(a, b)
|
||||
})
|
||||
|
||||
glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks))
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *fil
|
|||
} else {
|
||||
fh.counter++
|
||||
}
|
||||
fh.entry = entry
|
||||
return fh
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/mem"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -14,11 +15,12 @@ var (
|
|||
type ActualChunkIndex int
|
||||
|
||||
type SwapFile struct {
|
||||
dir string
|
||||
file *os.File
|
||||
logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
|
||||
chunkSize int64
|
||||
freeActualChunkList []ActualChunkIndex
|
||||
dir string
|
||||
file *os.File
|
||||
logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
|
||||
logicToActualChunkIndexLock sync.Mutex
|
||||
chunkSize int64
|
||||
freeActualChunkList []ActualChunkIndex
|
||||
}
|
||||
|
||||
type SwapFileChunk struct {
|
||||
|
|
@ -52,6 +54,8 @@ func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapF
|
|||
return nil
|
||||
}
|
||||
}
|
||||
sf.logicToActualChunkIndexLock.Lock()
|
||||
defer sf.logicToActualChunkIndexLock.Unlock()
|
||||
actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
|
||||
if !found {
|
||||
if len(sf.freeActualChunkList) > 0 {
|
||||
|
|
@ -72,6 +76,9 @@ func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapF
|
|||
}
|
||||
|
||||
func (sc *SwapFileChunk) FreeResource() {
|
||||
sc.swapfile.logicToActualChunkIndexLock.Lock()
|
||||
defer sc.swapfile.logicToActualChunkIndexLock.Unlock()
|
||||
|
||||
sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
|
||||
delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -187,6 +187,9 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
|
|||
|
||||
func (up *UploadPipeline) Shutdown() {
|
||||
up.swapFile.FreeResource()
|
||||
|
||||
up.sealedChunksLock.Lock()
|
||||
defer up.sealedChunksLock.Unlock()
|
||||
for logicChunkIndex, sealedChunk := range up.sealedChunks {
|
||||
sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
|
||||
|
|
@ -59,6 +60,7 @@ type WFS struct {
|
|||
// https://dl.acm.org/doi/fullHtml/10.1145/3310148
|
||||
// follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go
|
||||
fuse.RawFileSystem
|
||||
mount_pb.UnimplementedSeaweedMountServer
|
||||
fs.Inode
|
||||
option *Option
|
||||
metaCache *meta_cache.MetaCache
|
||||
|
|
@ -129,6 +131,9 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
|
|||
}
|
||||
var found bool
|
||||
if fh, found = wfs.fhmap.FindFileHandle(inode); found {
|
||||
if fh.entry.Attributes == nil {
|
||||
fh.entry.Attributes = &filer_pb.FuseAttributes{}
|
||||
}
|
||||
return path, fh, fh.entry, fuse.OK
|
||||
}
|
||||
entry, status = wfs.maybeLoadEntry(path)
|
||||
|
|
|
|||
|
|
@ -98,15 +98,14 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
|
|||
}
|
||||
}
|
||||
|
||||
if mtime, ok := input.GetMTime(); ok {
|
||||
entry.Attributes.Mtime = mtime.Unix()
|
||||
}
|
||||
|
||||
if atime, ok := input.GetATime(); ok {
|
||||
entry.Attributes.Mtime = atime.Unix()
|
||||
}
|
||||
|
||||
entry.Attributes.Mtime = time.Now().Unix()
|
||||
if mtime, ok := input.GetMTime(); ok {
|
||||
entry.Attributes.Mtime = mtime.Unix()
|
||||
}
|
||||
|
||||
out.AttrValid = 1
|
||||
wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHa
|
|||
_, _, entry, status = wfs.maybeReadEntry(inode)
|
||||
if status == fuse.OK {
|
||||
fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry)
|
||||
fileHandle.entry = entry
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
17
weed/mount/weedfs_grpc_server.go
Normal file
17
weed/mount/weedfs_grpc_server.go
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
|
||||
)
|
||||
|
||||
func (wfs *WFS) Configure(ctx context.Context, request *mount_pb.ConfigureRequest) (*mount_pb.ConfigureResponse, error) {
|
||||
if wfs.option.Collection == "" {
|
||||
return nil, fmt.Errorf("mount quota only works when mounted to a new folder with a collection")
|
||||
}
|
||||
glog.V(0).Infof("quota changed from %d to %d", wfs.option.Quota, request.CollectionCapacity)
|
||||
wfs.option.Quota = request.GetCollectionCapacity()
|
||||
return &mount_pb.ConfigureResponse{}, nil
|
||||
}
|
||||
|
|
@ -10,12 +10,14 @@ import (
|
|||
|
||||
func (wfs *WFS) loopCheckQuota() {
|
||||
|
||||
if wfs.option.Quota <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
|
||||
time.Sleep(61 * time.Second)
|
||||
|
||||
if wfs.option.Quota <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
request := &filer_pb.StatisticsRequest{
|
||||
|
|
@ -47,7 +49,6 @@ func (wfs *WFS) loopCheckQuota() {
|
|||
glog.Warningf("read quota usage: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(61 * time.Second)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ gen:
|
|||
protoc filer.proto --go_out=./filer_pb --go-grpc_out=./filer_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
protoc remote.proto --go_out=./remote_pb --go-grpc_out=./remote_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
protoc iam.proto --go_out=./iam_pb --go-grpc_out=./iam_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
protoc mount.proto --go_out=./mount_pb --go-grpc_out=./mount_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
protoc messaging.proto --go_out=./messaging_pb --go-grpc_out=./messaging_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
|
||||
# protoc filer.proto --java_out=../../other/java/client/src/main/java
|
||||
cp filer.proto ../../other/java/client/src/main/proto
|
||||
|
|
|
|||
|
|
@ -48,6 +48,9 @@ service SeaweedFiler {
|
|||
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
|
||||
}
|
||||
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
|
||||
rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
|
||||
}
|
||||
|
||||
|
|
@ -311,6 +314,16 @@ message StatisticsResponse {
|
|||
uint64 file_count = 6;
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
|
||||
message GetFilerConfigurationRequest {
|
||||
}
|
||||
message GetFilerConfigurationResponse {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,6 @@ import (
|
|||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// SeaweedFilerClient is the client API for SeaweedFiler service.
|
||||
|
|
@ -31,6 +30,7 @@ type SeaweedFilerClient interface {
|
|||
CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error)
|
||||
DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
|
||||
Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
|
||||
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
|
||||
GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error)
|
||||
SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error)
|
||||
SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error)
|
||||
|
|
@ -212,6 +212,15 @@ func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsReque
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedFilerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
|
||||
out := new(PingResponse)
|
||||
err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/Ping", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) {
|
||||
out := new(GetFilerConfigurationResponse)
|
||||
err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, opts...)
|
||||
|
|
@ -369,6 +378,7 @@ type SeaweedFilerServer interface {
|
|||
CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error)
|
||||
DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
|
||||
Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
|
||||
Ping(context.Context, *PingRequest) (*PingResponse, error)
|
||||
GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error)
|
||||
SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error
|
||||
SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error
|
||||
|
|
@ -423,6 +433,9 @@ func (UnimplementedSeaweedFilerServer) DeleteCollection(context.Context, *Delete
|
|||
func (UnimplementedSeaweedFilerServer) Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Statistics not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedFilerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedFilerServer) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method GetFilerConfiguration not implemented")
|
||||
}
|
||||
|
|
@ -700,6 +713,24 @@ func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec
|
|||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _SeaweedFiler_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(PingRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedFilerServer).Ping(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/filer_pb.SeaweedFiler/Ping",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedFilerServer).Ping(ctx, req.(*PingRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(GetFilerConfigurationRequest)
|
||||
if err := dec(in); err != nil {
|
||||
|
|
@ -909,6 +940,10 @@ var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{
|
|||
MethodName: "Statistics",
|
||||
Handler: _SeaweedFiler_Statistics_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Ping",
|
||||
Handler: _SeaweedFiler_Ping_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "GetFilerConfiguration",
|
||||
Handler: _SeaweedFiler_GetFilerConfiguration_Handler,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
|
|
@ -206,7 +207,15 @@ func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption g
|
|||
|
||||
}
|
||||
|
||||
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
|
||||
func WithVolumeServerClient(streamingMode bool, volumeServer ServerAddress, grpcDialOption grpc.DialOption, fn func(client volume_server_pb.VolumeServerClient) error) error {
|
||||
return WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
|
||||
client := volume_server_pb.NewVolumeServerClient(grpcConnection)
|
||||
return fn(client)
|
||||
}, volumeServer.ToGrpcAddress(), grpcDialOption)
|
||||
|
||||
}
|
||||
|
||||
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
|
||||
|
||||
for _, masterGrpcAddress := range masterGrpcAddresses {
|
||||
err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.17.3
|
||||
// protoc-gen-go v1.28.0
|
||||
// protoc v3.19.4
|
||||
// source: iam.proto
|
||||
|
||||
package iam_pb
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import (
|
|||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.
|
||||
|
|
|
|||
|
|
@ -35,7 +35,14 @@ service Seaweed {
|
|||
}
|
||||
rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
|
||||
}
|
||||
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) {
|
||||
}
|
||||
rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) {
|
||||
}
|
||||
rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
|
@ -140,6 +147,8 @@ message VolumeLocation {
|
|||
string leader = 5; // optional when leader is not itself
|
||||
string data_center = 6; // optional when DataCenter is in use
|
||||
uint32 grpc_port = 7;
|
||||
repeated uint32 new_ec_vids = 8;
|
||||
repeated uint32 deleted_ec_vids = 9;
|
||||
}
|
||||
|
||||
message ClusterNodeUpdate {
|
||||
|
|
@ -281,6 +290,8 @@ message LookupEcVolumeResponse {
|
|||
|
||||
message VacuumVolumeRequest {
|
||||
float garbage_threshold = 1;
|
||||
uint32 volume_id = 2;
|
||||
string collection = 3;
|
||||
}
|
||||
message VacuumVolumeResponse {
|
||||
}
|
||||
|
|
@ -328,3 +339,39 @@ message ReleaseAdminTokenRequest {
|
|||
}
|
||||
message ReleaseAdminTokenResponse {
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
|
||||
message RaftAddServerRequest {
|
||||
string id = 1;
|
||||
string address = 2;
|
||||
bool voter = 3;
|
||||
}
|
||||
message RaftAddServerResponse {
|
||||
}
|
||||
|
||||
message RaftRemoveServerRequest {
|
||||
string id = 1;
|
||||
bool force = 2;
|
||||
}
|
||||
message RaftRemoveServerResponse {
|
||||
}
|
||||
|
||||
message RaftListClusterServersRequest {
|
||||
}
|
||||
message RaftListClusterServersResponse {
|
||||
message ClusterServers {
|
||||
string id = 1;
|
||||
string address = 2;
|
||||
string suffrage = 3; //
|
||||
}
|
||||
repeated ClusterServers cluster_servers = 1;
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,6 @@ import (
|
|||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// SeaweedClient is the client API for Seaweed service.
|
||||
|
|
@ -32,6 +31,10 @@ type SeaweedClient interface {
|
|||
ListClusterNodes(ctx context.Context, in *ListClusterNodesRequest, opts ...grpc.CallOption) (*ListClusterNodesResponse, error)
|
||||
LeaseAdminToken(ctx context.Context, in *LeaseAdminTokenRequest, opts ...grpc.CallOption) (*LeaseAdminTokenResponse, error)
|
||||
ReleaseAdminToken(ctx context.Context, in *ReleaseAdminTokenRequest, opts ...grpc.CallOption) (*ReleaseAdminTokenResponse, error)
|
||||
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
|
||||
RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error)
|
||||
RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error)
|
||||
RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error)
|
||||
}
|
||||
|
||||
type seaweedClient struct {
|
||||
|
|
@ -212,6 +215,42 @@ func (c *seaweedClient) ReleaseAdminToken(ctx context.Context, in *ReleaseAdminT
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
|
||||
out := new(PingResponse)
|
||||
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/Ping", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) RaftListClusterServers(ctx context.Context, in *RaftListClusterServersRequest, opts ...grpc.CallOption) (*RaftListClusterServersResponse, error) {
|
||||
out := new(RaftListClusterServersResponse)
|
||||
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftListClusterServers", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) RaftAddServer(ctx context.Context, in *RaftAddServerRequest, opts ...grpc.CallOption) (*RaftAddServerResponse, error) {
|
||||
out := new(RaftAddServerResponse)
|
||||
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftAddServer", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *seaweedClient) RaftRemoveServer(ctx context.Context, in *RaftRemoveServerRequest, opts ...grpc.CallOption) (*RaftRemoveServerResponse, error) {
|
||||
out := new(RaftRemoveServerResponse)
|
||||
err := c.cc.Invoke(ctx, "/master_pb.Seaweed/RaftRemoveServer", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeaweedServer is the server API for Seaweed service.
|
||||
// All implementations must embed UnimplementedSeaweedServer
|
||||
// for forward compatibility
|
||||
|
|
@ -230,6 +269,10 @@ type SeaweedServer interface {
|
|||
ListClusterNodes(context.Context, *ListClusterNodesRequest) (*ListClusterNodesResponse, error)
|
||||
LeaseAdminToken(context.Context, *LeaseAdminTokenRequest) (*LeaseAdminTokenResponse, error)
|
||||
ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error)
|
||||
Ping(context.Context, *PingRequest) (*PingResponse, error)
|
||||
RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error)
|
||||
RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error)
|
||||
RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error)
|
||||
mustEmbedUnimplementedSeaweedServer()
|
||||
}
|
||||
|
||||
|
|
@ -279,6 +322,18 @@ func (UnimplementedSeaweedServer) LeaseAdminToken(context.Context, *LeaseAdminTo
|
|||
func (UnimplementedSeaweedServer) ReleaseAdminToken(context.Context, *ReleaseAdminTokenRequest) (*ReleaseAdminTokenResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReleaseAdminToken not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) RaftListClusterServers(context.Context, *RaftListClusterServersRequest) (*RaftListClusterServersResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method RaftListClusterServers not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) RaftAddServer(context.Context, *RaftAddServerRequest) (*RaftAddServerResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method RaftAddServer not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) RaftRemoveServer(context.Context, *RaftRemoveServerRequest) (*RaftRemoveServerResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method RaftRemoveServer not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedServer) mustEmbedUnimplementedSeaweedServer() {}
|
||||
|
||||
// UnsafeSeaweedServer may be embedded to opt out of forward compatibility for this service.
|
||||
|
|
@ -560,6 +615,78 @@ func _Seaweed_ReleaseAdminToken_Handler(srv interface{}, ctx context.Context, de
|
|||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(PingRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedServer).Ping(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/master_pb.Seaweed/Ping",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedServer).Ping(ctx, req.(*PingRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_RaftListClusterServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RaftListClusterServersRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedServer).RaftListClusterServers(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/master_pb.Seaweed/RaftListClusterServers",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedServer).RaftListClusterServers(ctx, req.(*RaftListClusterServersRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_RaftAddServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RaftAddServerRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedServer).RaftAddServer(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/master_pb.Seaweed/RaftAddServer",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedServer).RaftAddServer(ctx, req.(*RaftAddServerRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _Seaweed_RaftRemoveServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RaftRemoveServerRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedServer).RaftRemoveServer(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/master_pb.Seaweed/RaftRemoveServer",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedServer).RaftRemoveServer(ctx, req.(*RaftRemoveServerRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// Seaweed_ServiceDesc is the grpc.ServiceDesc for Seaweed service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
|
|
@ -615,6 +742,22 @@ var Seaweed_ServiceDesc = grpc.ServiceDesc{
|
|||
MethodName: "ReleaseAdminToken",
|
||||
Handler: _Seaweed_ReleaseAdminToken_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Ping",
|
||||
Handler: _Seaweed_Ping_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "RaftListClusterServers",
|
||||
Handler: _Seaweed_RaftListClusterServers_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "RaftAddServer",
|
||||
Handler: _Seaweed_RaftAddServer_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "RaftRemoveServer",
|
||||
Handler: _Seaweed_RaftRemoveServer_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.17.3
|
||||
// protoc-gen-go v1.28.0
|
||||
// protoc v3.19.4
|
||||
// source: messaging.proto
|
||||
|
||||
package messaging_pb
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import (
|
|||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// SeaweedMessagingClient is the client API for SeaweedMessaging service.
|
||||
|
|
|
|||
25
weed/pb/mount.proto
Normal file
25
weed/pb/mount.proto
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package messaging_pb;
|
||||
|
||||
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/mount_pb";
|
||||
option java_package = "seaweedfs.client";
|
||||
option java_outer_classname = "MountProto";
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
service SeaweedMount {
|
||||
|
||||
rpc Configure (ConfigureRequest) returns (ConfigureResponse) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message ConfigureRequest {
|
||||
int64 collection_capacity = 1;
|
||||
}
|
||||
|
||||
message ConfigureResponse {
|
||||
}
|
||||
208
weed/pb/mount_pb/mount.pb.go
Normal file
208
weed/pb/mount_pb/mount.pb.go
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.28.0
|
||||
// protoc v3.19.4
|
||||
// source: mount.proto
|
||||
|
||||
package mount_pb
|
||||
|
||||
import (
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// Verify that this generated code is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
|
||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
type ConfigureRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
CollectionCapacity int64 `protobuf:"varint,1,opt,name=collection_capacity,json=collectionCapacity,proto3" json:"collection_capacity,omitempty"`
|
||||
}
|
||||
|
||||
func (x *ConfigureRequest) Reset() {
|
||||
*x = ConfigureRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_mount_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *ConfigureRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ConfigureRequest) ProtoMessage() {}
|
||||
|
||||
func (x *ConfigureRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_mount_proto_msgTypes[0]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ConfigureRequest.ProtoReflect.Descriptor instead.
|
||||
func (*ConfigureRequest) Descriptor() ([]byte, []int) {
|
||||
return file_mount_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
func (x *ConfigureRequest) GetCollectionCapacity() int64 {
|
||||
if x != nil {
|
||||
return x.CollectionCapacity
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type ConfigureResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
}
|
||||
|
||||
func (x *ConfigureResponse) Reset() {
|
||||
*x = ConfigureResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_mount_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *ConfigureResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ConfigureResponse) ProtoMessage() {}
|
||||
|
||||
func (x *ConfigureResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_mount_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ConfigureResponse.ProtoReflect.Descriptor instead.
|
||||
func (*ConfigureResponse) Descriptor() ([]byte, []int) {
|
||||
return file_mount_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
var File_mount_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_mount_proto_rawDesc = []byte{
|
||||
0x0a, 0x0b, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d,
|
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x22, 0x43, 0x0a, 0x10, 0x43,
|
||||
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
|
||||
0x2f, 0x0a, 0x13, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x61,
|
||||
0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x63, 0x6f,
|
||||
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79,
|
||||
0x22, 0x13, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x5e, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
|
||||
0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x4e, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
|
||||
0x72, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64,
|
||||
0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x4d, 0x6f, 0x75, 0x6e, 0x74,
|
||||
0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77,
|
||||
0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x6f,
|
||||
0x75, 0x6e, 0x74, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
file_mount_proto_rawDescOnce sync.Once
|
||||
file_mount_proto_rawDescData = file_mount_proto_rawDesc
|
||||
)
|
||||
|
||||
func file_mount_proto_rawDescGZIP() []byte {
|
||||
file_mount_proto_rawDescOnce.Do(func() {
|
||||
file_mount_proto_rawDescData = protoimpl.X.CompressGZIP(file_mount_proto_rawDescData)
|
||||
})
|
||||
return file_mount_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_mount_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
|
||||
var file_mount_proto_goTypes = []interface{}{
|
||||
(*ConfigureRequest)(nil), // 0: messaging_pb.ConfigureRequest
|
||||
(*ConfigureResponse)(nil), // 1: messaging_pb.ConfigureResponse
|
||||
}
|
||||
var file_mount_proto_depIdxs = []int32{
|
||||
0, // 0: messaging_pb.SeaweedMount.Configure:input_type -> messaging_pb.ConfigureRequest
|
||||
1, // 1: messaging_pb.SeaweedMount.Configure:output_type -> messaging_pb.ConfigureResponse
|
||||
1, // [1:2] is the sub-list for method output_type
|
||||
0, // [0:1] is the sub-list for method input_type
|
||||
0, // [0:0] is the sub-list for extension type_name
|
||||
0, // [0:0] is the sub-list for extension extendee
|
||||
0, // [0:0] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_mount_proto_init() }
|
||||
func file_mount_proto_init() {
|
||||
if File_mount_proto != nil {
|
||||
return
|
||||
}
|
||||
if !protoimpl.UnsafeEnabled {
|
||||
file_mount_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*ConfigureRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_mount_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*ConfigureResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_mount_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 2,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
GoTypes: file_mount_proto_goTypes,
|
||||
DependencyIndexes: file_mount_proto_depIdxs,
|
||||
MessageInfos: file_mount_proto_msgTypes,
|
||||
}.Build()
|
||||
File_mount_proto = out.File
|
||||
file_mount_proto_rawDesc = nil
|
||||
file_mount_proto_goTypes = nil
|
||||
file_mount_proto_depIdxs = nil
|
||||
}
|
||||
100
weed/pb/mount_pb/mount_grpc.pb.go
Normal file
100
weed/pb/mount_pb/mount_grpc.pb.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
|
||||
package mount_pb
|
||||
|
||||
import (
|
||||
context "context"
|
||||
grpc "google.golang.org/grpc"
|
||||
codes "google.golang.org/grpc/codes"
|
||||
status "google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// SeaweedMountClient is the client API for SeaweedMount service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
type SeaweedMountClient interface {
|
||||
Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error)
|
||||
}
|
||||
|
||||
type seaweedMountClient struct {
|
||||
cc grpc.ClientConnInterface
|
||||
}
|
||||
|
||||
func NewSeaweedMountClient(cc grpc.ClientConnInterface) SeaweedMountClient {
|
||||
return &seaweedMountClient{cc}
|
||||
}
|
||||
|
||||
func (c *seaweedMountClient) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) {
|
||||
out := new(ConfigureResponse)
|
||||
err := c.cc.Invoke(ctx, "/messaging_pb.SeaweedMount/Configure", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeaweedMountServer is the server API for SeaweedMount service.
|
||||
// All implementations must embed UnimplementedSeaweedMountServer
|
||||
// for forward compatibility
|
||||
type SeaweedMountServer interface {
|
||||
Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error)
|
||||
mustEmbedUnimplementedSeaweedMountServer()
|
||||
}
|
||||
|
||||
// UnimplementedSeaweedMountServer must be embedded to have forward compatible implementations.
|
||||
type UnimplementedSeaweedMountServer struct {
|
||||
}
|
||||
|
||||
func (UnimplementedSeaweedMountServer) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented")
|
||||
}
|
||||
func (UnimplementedSeaweedMountServer) mustEmbedUnimplementedSeaweedMountServer() {}
|
||||
|
||||
// UnsafeSeaweedMountServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to SeaweedMountServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeSeaweedMountServer interface {
|
||||
mustEmbedUnimplementedSeaweedMountServer()
|
||||
}
|
||||
|
||||
func RegisterSeaweedMountServer(s grpc.ServiceRegistrar, srv SeaweedMountServer) {
|
||||
s.RegisterService(&SeaweedMount_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _SeaweedMount_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ConfigureRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(SeaweedMountServer).Configure(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/messaging_pb.SeaweedMount/Configure",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(SeaweedMountServer).Configure(ctx, req.(*ConfigureRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// SeaweedMount_ServiceDesc is the grpc.ServiceDesc for SeaweedMount service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var SeaweedMount_ServiceDesc = grpc.ServiceDesc{
|
||||
ServiceName: "messaging_pb.SeaweedMount",
|
||||
HandlerType: (*SeaweedMountServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "Configure",
|
||||
Handler: _SeaweedMount_Configure_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
Metadata: "mount.proto",
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.6.1
|
||||
// protoc-gen-go v1.28.0
|
||||
// protoc v3.19.4
|
||||
// source: remote.proto
|
||||
|
||||
package remote_pb
|
||||
|
|
|
|||
|
|
@ -86,6 +86,14 @@ func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) {
|
|||
return
|
||||
}
|
||||
|
||||
func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) {
|
||||
addresses = make(map[string]ServerAddress)
|
||||
for _, address := range sa.ToAddresses() {
|
||||
addresses[string(address)] = address
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (sa ServerAddresses) ToAddressStrings() (addresses []string) {
|
||||
parts := strings.Split(string(sa), ",")
|
||||
for _, address := range parts {
|
||||
|
|
@ -101,6 +109,13 @@ func ToAddressStrings(addresses []ServerAddress) []string {
|
|||
}
|
||||
return strings
|
||||
}
|
||||
func ToAddressStringsFromMap(addresses map[string]ServerAddress) []string {
|
||||
var strings []string
|
||||
for _, addr := range addresses {
|
||||
strings = append(strings, string(addr))
|
||||
}
|
||||
return strings
|
||||
}
|
||||
func FromAddressStrings(strings []string) []ServerAddress {
|
||||
var addresses []ServerAddress
|
||||
for _, addr := range strings {
|
||||
|
|
|
|||
|
|
@ -107,6 +107,10 @@ service VolumeServer {
|
|||
|
||||
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
|
||||
}
|
||||
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
|
@ -573,3 +577,13 @@ message VolumeNeedleStatusResponse {
|
|||
uint32 crc = 5;
|
||||
string ttl = 6;
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,7 +11,6 @@ import (
|
|||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.32.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion7
|
||||
|
||||
// VolumeServerClient is the client API for VolumeServer service.
|
||||
|
|
@ -64,6 +63,7 @@ type VolumeServerClient interface {
|
|||
// <experimental> query
|
||||
Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error)
|
||||
VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error)
|
||||
Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error)
|
||||
}
|
||||
|
||||
type volumeServerClient struct {
|
||||
|
|
@ -664,6 +664,15 @@ func (c *volumeServerClient) VolumeNeedleStatus(ctx context.Context, in *VolumeN
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) {
|
||||
out := new(PingResponse)
|
||||
err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/Ping", in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VolumeServerServer is the server API for VolumeServer service.
|
||||
// All implementations must embed UnimplementedVolumeServerServer
|
||||
// for forward compatibility
|
||||
|
|
@ -714,6 +723,7 @@ type VolumeServerServer interface {
|
|||
// <experimental> query
|
||||
Query(*QueryRequest, VolumeServer_QueryServer) error
|
||||
VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error)
|
||||
Ping(context.Context, *PingRequest) (*PingResponse, error)
|
||||
mustEmbedUnimplementedVolumeServerServer()
|
||||
}
|
||||
|
||||
|
|
@ -841,6 +851,9 @@ func (UnimplementedVolumeServerServer) Query(*QueryRequest, VolumeServer_QuerySe
|
|||
func (UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {}
|
||||
|
||||
// UnsafeVolumeServerServer may be embedded to opt out of forward compatibility for this service.
|
||||
|
|
@ -1604,6 +1617,24 @@ func _VolumeServer_VolumeNeedleStatus_Handler(srv interface{}, ctx context.Conte
|
|||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(PingRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServerServer).Ping(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: "/volume_server_pb.VolumeServer/Ping",
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServerServer).Ping(ctx, req.(*PingRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// VolumeServer_ServiceDesc is the grpc.ServiceDesc for VolumeServer service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
|
|
@ -1731,6 +1762,10 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
|
|||
MethodName: "VolumeNeedleStatus",
|
||||
Handler: _VolumeServer_VolumeNeedleStatus_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "Ping",
|
||||
Handler: _VolumeServer_Ping_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{
|
||||
{
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
const slash = "/"
|
||||
|
||||
func ParseLocationName(remote string) (locationName string) {
|
||||
if strings.HasSuffix(string(remote), "/") {
|
||||
remote = remote[:len(remote)-1]
|
||||
}
|
||||
parts := strings.SplitN(string(remote), "/", 2)
|
||||
remote = strings.TrimSuffix(remote, slash)
|
||||
parts := strings.SplitN(remote, slash, 2)
|
||||
if len(parts) >= 1 {
|
||||
return parts[0]
|
||||
}
|
||||
|
|
@ -25,35 +25,31 @@ func ParseLocationName(remote string) (locationName string) {
|
|||
|
||||
func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
|
||||
loc = &remote_pb.RemoteStorageLocation{}
|
||||
if strings.HasSuffix(string(remote), "/") {
|
||||
remote = remote[:len(remote)-1]
|
||||
}
|
||||
parts := strings.SplitN(string(remote), "/", 3)
|
||||
remote = strings.TrimSuffix(remote, slash)
|
||||
parts := strings.SplitN(remote, slash, 3)
|
||||
if len(parts) >= 1 {
|
||||
loc.Name = parts[0]
|
||||
}
|
||||
if len(parts) >= 2 {
|
||||
loc.Bucket = parts[1]
|
||||
}
|
||||
loc.Path = string(remote[len(loc.Name)+1+len(loc.Bucket):])
|
||||
loc.Path = remote[len(loc.Name)+1+len(loc.Bucket):]
|
||||
if loc.Path == "" {
|
||||
loc.Path = "/"
|
||||
loc.Path = slash
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
|
||||
loc = &remote_pb.RemoteStorageLocation{}
|
||||
if strings.HasSuffix(string(remote), "/") {
|
||||
remote = remote[:len(remote)-1]
|
||||
}
|
||||
parts := strings.SplitN(string(remote), "/", 2)
|
||||
remote = strings.TrimSuffix(remote, slash)
|
||||
parts := strings.SplitN(remote, slash, 2)
|
||||
if len(parts) >= 1 {
|
||||
loc.Name = parts[0]
|
||||
}
|
||||
loc.Path = string(remote[len(loc.Name):])
|
||||
loc.Path = remote[len(loc.Name):]
|
||||
if loc.Path == "" {
|
||||
loc.Path = "/"
|
||||
loc.Path = slash
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
package s3api
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
"golang.org/x/exp/slices"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
|
@ -12,7 +14,6 @@ import (
|
|||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
|
@ -28,8 +29,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
|
|||
|
||||
glog.V(2).Infof("createMultipartUpload input %v", input)
|
||||
|
||||
uploadId, _ := uuid.NewRandom()
|
||||
uploadIdString := uploadId.String()
|
||||
uploadIdString := s3a.generateUploadID(*input.Key)
|
||||
|
||||
if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
|
|
@ -68,8 +68,8 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
|
|||
glog.V(2).Infof("completeMultipartUpload input %v", input)
|
||||
|
||||
completedParts := parts.Parts
|
||||
sort.Slice(completedParts, func(i, j int) bool {
|
||||
return completedParts[i].PartNumber < completedParts[j].PartNumber
|
||||
slices.SortFunc(completedParts, func(a, b CompletedPart) bool {
|
||||
return a.PartNumber < b.PartNumber
|
||||
})
|
||||
|
||||
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
|
||||
|
|
@ -93,10 +93,15 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
|
|||
|
||||
for _, entry := range entries {
|
||||
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
|
||||
_, found := findByPartNumber(entry.Name, completedParts)
|
||||
partETag, found := findByPartNumber(entry.Name, completedParts)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
entryETag := hex.EncodeToString(entry.Attributes.GetMd5())
|
||||
if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag {
|
||||
glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag)
|
||||
return nil, s3err.ErrInvalidPart
|
||||
}
|
||||
for _, chunk := range entry.Chunks {
|
||||
p := &filer_pb.FileChunk{
|
||||
FileId: chunk.GetFileIdString(),
|
||||
|
|
@ -175,7 +180,15 @@ func findByPartNumber(fileName string, parts []CompletedPart) (etag string, foun
|
|||
if parts[x].PartNumber != partNumber {
|
||||
return
|
||||
}
|
||||
return parts[x].ETag, true
|
||||
y := 0
|
||||
for i, part := range parts[x:] {
|
||||
if part.PartNumber == partNumber {
|
||||
y = i
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return parts[x+y].ETag, true
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,10 @@ func Test_findByPartNumber(t *testing.T) {
|
|||
ETag: "xxx",
|
||||
PartNumber: 1,
|
||||
},
|
||||
CompletedPart{
|
||||
ETag: "lll",
|
||||
PartNumber: 1,
|
||||
},
|
||||
CompletedPart{
|
||||
ETag: "yyy",
|
||||
PartNumber: 3,
|
||||
|
|
@ -83,7 +87,7 @@ func Test_findByPartNumber(t *testing.T) {
|
|||
"0001.part",
|
||||
parts,
|
||||
},
|
||||
"xxx",
|
||||
"lll",
|
||||
true,
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package s3api
|
|||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
|
|
@ -134,6 +135,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
|
|||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/" + bucket)
|
||||
writeSuccessResponseEmpty(w, r)
|
||||
}
|
||||
|
||||
|
|
@ -148,6 +150,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
|
||||
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
if !s3a.option.AllowDeleteBucketNotEmpty {
|
||||
entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list bucket %s: %v", bucket, err)
|
||||
}
|
||||
if len(entries) > 0 {
|
||||
return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code)
|
||||
}
|
||||
}
|
||||
|
||||
// delete collection
|
||||
deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
|
||||
|
|
@ -162,6 +173,15 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
|
|||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
s3ErrorCode := s3err.ErrInternalError
|
||||
if err.Error() == s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code {
|
||||
s3ErrorCode = s3err.ErrBucketNotEmpty
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, s3ErrorCode)
|
||||
return
|
||||
}
|
||||
|
||||
err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)
|
||||
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ import (
|
|||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/mem"
|
||||
"golang.org/x/exp/slices"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -27,6 +27,10 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
deleteMultipleObjectsLimmit = 1000
|
||||
)
|
||||
|
||||
func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser {
|
||||
mimeBuffer := make([]byte, 512)
|
||||
size, _ := dataReader.Read(mimeBuffer)
|
||||
|
|
@ -59,7 +63,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
if r.Header.Get("Expires") != "" {
|
||||
if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrMalformedExpires)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -217,6 +221,11 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
|||
return
|
||||
}
|
||||
|
||||
if len(deleteObjects.Objects) > deleteMultipleObjectsLimmit {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxDeleteObjects)
|
||||
return
|
||||
}
|
||||
|
||||
var deletedObjects []ObjectIdentifier
|
||||
var deleteErrors []DeleteError
|
||||
var auditLog *s3err.AccessLog
|
||||
|
|
@ -281,8 +290,8 @@ func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerCli
|
|||
for dir, _ := range directoriesWithDeletion {
|
||||
allDirs = append(allDirs, dir)
|
||||
}
|
||||
sort.Slice(allDirs, func(i, j int) bool {
|
||||
return len(allDirs[i]) > len(allDirs[j])
|
||||
slices.SortFunc(allDirs, func(a, b string) bool {
|
||||
return len(a) > len(b)
|
||||
})
|
||||
newDirectoriesWithDeletion = make(map[string]int)
|
||||
for _, dir := range allDirs {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package s3api
|
|||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||
|
|
@ -70,6 +71,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
|||
|
||||
// Get upload id.
|
||||
uploadID, _, _, _ := getObjectResources(r.URL.Query())
|
||||
err := s3a.checkUploadId(object, uploadID)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
return
|
||||
}
|
||||
|
||||
response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
|
|
@ -94,6 +100,11 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
|
|||
|
||||
// Get upload id.
|
||||
uploadID, _, _, _ := getObjectResources(r.URL.Query())
|
||||
err := s3a.checkUploadId(object, uploadID)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
return
|
||||
}
|
||||
|
||||
response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
|
|
@ -165,6 +176,12 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
|
|||
return
|
||||
}
|
||||
|
||||
err := s3a.checkUploadId(object, uploadID)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
return
|
||||
}
|
||||
|
||||
response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: objectKey(aws.String(object)),
|
||||
|
|
@ -186,11 +203,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
|
|||
|
||||
// PutObjectPartHandler - Put an object part in a multipart upload.
|
||||
func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := xhttp.GetBucketAndObject(r)
|
||||
bucket, object := xhttp.GetBucketAndObject(r)
|
||||
|
||||
uploadID := r.URL.Query().Get("uploadId")
|
||||
exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
|
||||
if !exists {
|
||||
err := s3a.checkUploadId(object, uploadID)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
return
|
||||
}
|
||||
|
|
@ -250,6 +267,27 @@ func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
|
|||
return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
|
||||
}
|
||||
|
||||
// Generate uploadID hash string from object
|
||||
func (s3a *S3ApiServer) generateUploadID(object string) string {
|
||||
if strings.HasPrefix(object, "/") {
|
||||
object = object[1:]
|
||||
}
|
||||
h := sha1.New()
|
||||
h.Write([]byte(object))
|
||||
return fmt.Sprintf("%x", h.Sum(nil))
|
||||
}
|
||||
|
||||
//Check object name and uploadID when processing multipart uploading
|
||||
func (s3a *S3ApiServer) checkUploadId(object string, id string) error {
|
||||
|
||||
hash := s3a.generateUploadID(object)
|
||||
if hash != id {
|
||||
glog.Errorf("object %s and uploadID %s are not matched", object, id)
|
||||
return fmt.Errorf("object %s and uploadID %s are not matched", object, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse bucket url queries for ?uploads
|
||||
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
|
||||
prefix = values.Get("prefix")
|
||||
|
|
|
|||
|
|
@ -19,14 +19,15 @@ import (
|
|||
)
|
||||
|
||||
type S3ApiServerOption struct {
|
||||
Filer pb.ServerAddress
|
||||
Port int
|
||||
Config string
|
||||
DomainName string
|
||||
BucketsPath string
|
||||
GrpcDialOption grpc.DialOption
|
||||
AllowEmptyFolder bool
|
||||
LocalFilerSocket *string
|
||||
Filer pb.ServerAddress
|
||||
Port int
|
||||
Config string
|
||||
DomainName string
|
||||
BucketsPath string
|
||||
GrpcDialOption grpc.DialOption
|
||||
AllowEmptyFolder bool
|
||||
AllowDeleteBucketNotEmpty bool
|
||||
LocalFilerSocket *string
|
||||
}
|
||||
|
||||
type S3ApiServer struct {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ const (
|
|||
ErrInvalidMaxKeys
|
||||
ErrInvalidMaxUploads
|
||||
ErrInvalidMaxParts
|
||||
ErrInvalidMaxDeleteObjects
|
||||
ErrInvalidPartNumberMarker
|
||||
ErrInvalidPart
|
||||
ErrInternalError
|
||||
|
|
@ -157,6 +158,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
|
|||
Description: "Argument max-parts must be an integer between 0 and 2147483647",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidMaxDeleteObjects: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Argument objects can contain a list of up to 1000 keys",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidPartNumberMarker: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Argument partNumberMarker must be an integer.",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package weed_server
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
|
@ -357,128 +356,3 @@ func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.Delet
|
|||
|
||||
return &filer_pb.DeleteCollectionResponse{}, err
|
||||
}
|
||||
|
||||
func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
|
||||
|
||||
var output *master_pb.StatisticsResponse
|
||||
|
||||
err = fs.filer.MasterClient.WithClient(false, func(masterClient master_pb.SeaweedClient) error {
|
||||
grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{
|
||||
Replication: req.Replication,
|
||||
Collection: req.Collection,
|
||||
Ttl: req.Ttl,
|
||||
DiskType: req.DiskType,
|
||||
})
|
||||
if grpcErr != nil {
|
||||
return grpcErr
|
||||
}
|
||||
|
||||
output = grpcResponse
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &filer_pb.StatisticsResponse{
|
||||
TotalSize: output.TotalSize,
|
||||
UsedSize: output.UsedSize,
|
||||
FileCount: output.FileCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
|
||||
|
||||
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
|
||||
|
||||
t := &filer_pb.GetFilerConfigurationResponse{
|
||||
Masters: pb.ToAddressStrings(fs.option.Masters),
|
||||
Collection: fs.option.Collection,
|
||||
Replication: fs.option.DefaultReplication,
|
||||
MaxMb: uint32(fs.option.MaxMB),
|
||||
DirBuckets: fs.filer.DirBucketsPath,
|
||||
Cipher: fs.filer.Cipher,
|
||||
Signature: fs.filer.Signature,
|
||||
MetricsAddress: fs.metricsAddress,
|
||||
MetricsIntervalSec: int32(fs.metricsIntervalSec),
|
||||
Version: util.Version(),
|
||||
ClusterId: string(clusterId),
|
||||
}
|
||||
|
||||
glog.V(4).Infof("GetFilerConfiguration: %v", t)
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error {
|
||||
|
||||
req, err := stream.Recv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clientName := util.JoinHostPort(req.Name, int(req.GrpcPort))
|
||||
m := make(map[string]bool)
|
||||
for _, tp := range req.Resources {
|
||||
m[tp] = true
|
||||
}
|
||||
fs.brokersLock.Lock()
|
||||
fs.brokers[clientName] = m
|
||||
glog.V(0).Infof("+ broker %v", clientName)
|
||||
fs.brokersLock.Unlock()
|
||||
|
||||
defer func() {
|
||||
fs.brokersLock.Lock()
|
||||
delete(fs.brokers, clientName)
|
||||
glog.V(0).Infof("- broker %v: %v", clientName, err)
|
||||
fs.brokersLock.Unlock()
|
||||
}()
|
||||
|
||||
for {
|
||||
if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil {
|
||||
glog.V(0).Infof("send broker %v: %+v", clientName, err)
|
||||
return err
|
||||
}
|
||||
// println("replied")
|
||||
|
||||
if _, err := stream.Recv(); err != nil {
|
||||
glog.V(0).Infof("recv broker %v: %v", clientName, err)
|
||||
return err
|
||||
}
|
||||
// println("received")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) {
|
||||
|
||||
resp = &filer_pb.LocateBrokerResponse{}
|
||||
|
||||
fs.brokersLock.Lock()
|
||||
defer fs.brokersLock.Unlock()
|
||||
|
||||
var localBrokers []*filer_pb.LocateBrokerResponse_Resource
|
||||
|
||||
for b, m := range fs.brokers {
|
||||
if _, found := m[req.Resource]; found {
|
||||
resp.Found = true
|
||||
resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{
|
||||
{
|
||||
GrpcAddresses: b,
|
||||
ResourceCount: int32(len(m)),
|
||||
},
|
||||
}
|
||||
return
|
||||
}
|
||||
localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{
|
||||
GrpcAddresses: b,
|
||||
ResourceCount: int32(len(m)),
|
||||
})
|
||||
}
|
||||
|
||||
resp.Resources = localBrokers
|
||||
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
|
|
|||
177
weed/server/filer_grpc_server_admin.go
Normal file
177
weed/server/filer_grpc_server_admin.go
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/cluster"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
|
||||
|
||||
var output *master_pb.StatisticsResponse
|
||||
|
||||
err = fs.filer.MasterClient.WithClient(false, func(masterClient master_pb.SeaweedClient) error {
|
||||
grpcResponse, grpcErr := masterClient.Statistics(context.Background(), &master_pb.StatisticsRequest{
|
||||
Replication: req.Replication,
|
||||
Collection: req.Collection,
|
||||
Ttl: req.Ttl,
|
||||
DiskType: req.DiskType,
|
||||
})
|
||||
if grpcErr != nil {
|
||||
return grpcErr
|
||||
}
|
||||
|
||||
output = grpcResponse
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &filer_pb.StatisticsResponse{
|
||||
TotalSize: output.TotalSize,
|
||||
UsedSize: output.UsedSize,
|
||||
FileCount: output.FileCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) Ping(ctx context.Context, req *filer_pb.PingRequest) (resp *filer_pb.PingResponse, pingErr error) {
|
||||
resp = &filer_pb.PingResponse{
|
||||
StartTimeNs: time.Now().UnixNano(),
|
||||
}
|
||||
if req.TargetType == cluster.FilerType {
|
||||
pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if req.TargetType == cluster.VolumeServerType {
|
||||
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if req.TargetType == cluster.MasterType {
|
||||
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), fs.grpcDialOption, func(client master_pb.SeaweedClient) error {
|
||||
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if pingErr != nil {
|
||||
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
|
||||
}
|
||||
resp.StopTimeNs = time.Now().UnixNano()
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
|
||||
|
||||
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
|
||||
|
||||
t := &filer_pb.GetFilerConfigurationResponse{
|
||||
Masters: pb.ToAddressStringsFromMap(fs.option.Masters),
|
||||
Collection: fs.option.Collection,
|
||||
Replication: fs.option.DefaultReplication,
|
||||
MaxMb: uint32(fs.option.MaxMB),
|
||||
DirBuckets: fs.filer.DirBucketsPath,
|
||||
Cipher: fs.filer.Cipher,
|
||||
Signature: fs.filer.Signature,
|
||||
MetricsAddress: fs.metricsAddress,
|
||||
MetricsIntervalSec: int32(fs.metricsIntervalSec),
|
||||
Version: util.Version(),
|
||||
ClusterId: string(clusterId),
|
||||
}
|
||||
|
||||
glog.V(4).Infof("GetFilerConfiguration: %v", t)
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error {
|
||||
|
||||
req, err := stream.Recv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clientName := util.JoinHostPort(req.Name, int(req.GrpcPort))
|
||||
m := make(map[string]bool)
|
||||
for _, tp := range req.Resources {
|
||||
m[tp] = true
|
||||
}
|
||||
fs.brokersLock.Lock()
|
||||
fs.brokers[clientName] = m
|
||||
glog.V(0).Infof("+ broker %v", clientName)
|
||||
fs.brokersLock.Unlock()
|
||||
|
||||
defer func() {
|
||||
fs.brokersLock.Lock()
|
||||
delete(fs.brokers, clientName)
|
||||
glog.V(0).Infof("- broker %v: %v", clientName, err)
|
||||
fs.brokersLock.Unlock()
|
||||
}()
|
||||
|
||||
for {
|
||||
if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil {
|
||||
glog.V(0).Infof("send broker %v: %+v", clientName, err)
|
||||
return err
|
||||
}
|
||||
// println("replied")
|
||||
|
||||
if _, err := stream.Recv(); err != nil {
|
||||
glog.V(0).Infof("recv broker %v: %v", clientName, err)
|
||||
return err
|
||||
}
|
||||
// println("received")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) {
|
||||
|
||||
resp = &filer_pb.LocateBrokerResponse{}
|
||||
|
||||
fs.brokersLock.Lock()
|
||||
defer fs.brokersLock.Unlock()
|
||||
|
||||
var localBrokers []*filer_pb.LocateBrokerResponse_Resource
|
||||
|
||||
for b, m := range fs.brokers {
|
||||
if _, found := m[req.Resource]; found {
|
||||
resp.Found = true
|
||||
resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{
|
||||
{
|
||||
GrpcAddresses: b,
|
||||
ResourceCount: int32(len(m)),
|
||||
},
|
||||
}
|
||||
return
|
||||
}
|
||||
localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{
|
||||
GrpcAddresses: b,
|
||||
ResourceCount: int32(len(m)),
|
||||
})
|
||||
}
|
||||
|
||||
resp.Resources = localBrokers
|
||||
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
|
@ -48,7 +48,7 @@ import (
|
|||
)
|
||||
|
||||
type FilerOption struct {
|
||||
Masters []pb.ServerAddress
|
||||
Masters map[string]pb.ServerAddress
|
||||
Collection string
|
||||
DefaultReplication string
|
||||
DisableDirListing bool
|
||||
|
|
|
|||
|
|
@ -46,8 +46,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
path = ""
|
||||
}
|
||||
|
||||
emptyFolder := true
|
||||
if len(entries) > 0 {
|
||||
lastFileName = entries[len(entries)-1].Name()
|
||||
emptyFolder = false
|
||||
}
|
||||
|
||||
glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries))
|
||||
|
|
@ -59,12 +61,14 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
Limit int
|
||||
LastFileName string
|
||||
ShouldDisplayLoadMore bool
|
||||
EmptyFolder bool
|
||||
}{
|
||||
path,
|
||||
entries,
|
||||
limit,
|
||||
lastFileName,
|
||||
shouldDisplayLoadMore,
|
||||
emptyFolder,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -76,6 +80,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
Limit int
|
||||
LastFileName string
|
||||
ShouldDisplayLoadMore bool
|
||||
EmptyFolder bool
|
||||
}{
|
||||
path,
|
||||
ui.ToBreadcrumb(path),
|
||||
|
|
@ -83,5 +88,6 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
limit,
|
||||
lastFileName,
|
||||
shouldDisplayLoadMore,
|
||||
emptyFolder,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,9 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque
|
|||
toDelete := strings.Split(r.URL.Query().Get("tagging"), ",")
|
||||
deletions := make(map[string]struct{})
|
||||
for _, deletion := range toDelete {
|
||||
deletions[deletion] = struct{}{}
|
||||
if deletion != "" {
|
||||
deletions[deletion] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// delete all tags or specific tags
|
||||
|
|
|
|||
|
|
@ -164,6 +164,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
}
|
||||
|
||||
var entry *filer.Entry
|
||||
var newChunks []*filer_pb.FileChunk
|
||||
var mergedChunks []*filer_pb.FileChunk
|
||||
|
||||
isAppend := isAppend(r)
|
||||
|
|
@ -186,7 +187,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
}
|
||||
entry.FileSize += uint64(chunkOffset)
|
||||
}
|
||||
mergedChunks = append(entry.Chunks, fileChunks...)
|
||||
newChunks = append(entry.Chunks, fileChunks...)
|
||||
|
||||
// TODO
|
||||
if len(entry.Content) > 0 {
|
||||
|
|
@ -196,7 +197,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
|
||||
} else {
|
||||
glog.V(4).Infoln("saving", path)
|
||||
mergedChunks = fileChunks
|
||||
newChunks = fileChunks
|
||||
entry = &filer.Entry{
|
||||
FullPath: util.FullPath(path),
|
||||
Attr: filer.Attr{
|
||||
|
|
@ -217,6 +218,13 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
}
|
||||
}
|
||||
|
||||
// maybe concatenate small chunks into one whole chunk
|
||||
mergedChunks, replyerr = fs.maybeMergeChunks(so, newChunks)
|
||||
if replyerr != nil {
|
||||
glog.V(0).Infof("merge chunks %s: %v", r.RequestURI, replyerr)
|
||||
mergedChunks = newChunks
|
||||
}
|
||||
|
||||
// maybe compact entry chunks
|
||||
mergedChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), mergedChunks)
|
||||
if replyerr != nil {
|
||||
|
|
|
|||
11
weed/server/filer_server_handlers_write_merge.go
Normal file
11
weed/server/filer_server_handlers_write_merge.go
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func (fs *FilerServer) maybeMergeChunks(so *operation.StorageOption, inputChunks []*filer_pb.FileChunk) (mergedChunks []*filer_pb.FileChunk, err error) {
|
||||
//TODO merge consecutive smaller chunks into a large chunk to reduce number of chunks
|
||||
return inputChunks, nil
|
||||
}
|
||||
|
|
@ -4,10 +4,10 @@ import (
|
|||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"golang.org/x/exp/slices"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -130,11 +130,9 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
|
|||
fs.filer.DeleteChunks(fileChunks)
|
||||
return nil, md5Hash, 0, uploadErr, nil
|
||||
}
|
||||
|
||||
sort.Slice(fileChunks, func(i, j int) bool {
|
||||
return fileChunks[i].Offset < fileChunks[j].Offset
|
||||
slices.SortFunc(fileChunks, func(a, b *filer_pb.FileChunk) bool {
|
||||
return a.Offset < b.Offset
|
||||
})
|
||||
|
||||
return fileChunks, md5Hash, chunkOffset, nil, smallContent
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,8 +15,12 @@ func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) {
|
|||
parts := strings.Split(fullpath, "/")
|
||||
|
||||
for i := 0; i < len(parts); i++ {
|
||||
name := parts[i]
|
||||
if name == "" {
|
||||
name = "/"
|
||||
}
|
||||
crumb := Breadcrumb{
|
||||
Name: parts[i] + " /",
|
||||
Name: name,
|
||||
Link: "/" + util.Join(parts[0:i+1]...),
|
||||
}
|
||||
if !strings.HasSuffix(crumb.Link, "/") {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
|
||||
#drop-area {
|
||||
border: 1px transparent;
|
||||
margin-top: 5px;
|
||||
}
|
||||
|
||||
#drop-area.highlight {
|
||||
|
|
@ -26,6 +27,12 @@
|
|||
border-radius: 2px;
|
||||
border: 1px solid #ccc;
|
||||
float: right;
|
||||
margin-left: 2px;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
label {
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
.button:hover {
|
||||
|
|
@ -36,6 +43,38 @@
|
|||
display: none;
|
||||
}
|
||||
|
||||
td, th {
|
||||
vertical-align: bottom;
|
||||
}
|
||||
|
||||
.table-hover > tbody > tr:hover > * > div.operations {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.table > tbody > tr {
|
||||
height: 39px;
|
||||
}
|
||||
|
||||
div.operations {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.footer {
|
||||
position: absolute;
|
||||
bottom: 0px;
|
||||
right: 5%;
|
||||
min-width: 25%;
|
||||
border-left: 1px solid #ccc;
|
||||
border-right: 1px solid #ccc;
|
||||
}
|
||||
|
||||
.add-files {
|
||||
font-size: 46px;
|
||||
text-align: center;
|
||||
border: 1px dashed #999;
|
||||
padding-bottom: 9px;
|
||||
margin: 0 2px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -48,12 +87,21 @@
|
|||
</div>
|
||||
<div class="row">
|
||||
<div>
|
||||
<div class="btn-group btn-group-sm pull-right" role="group" style="margin-top:3px;">
|
||||
<label class="btn btn-default" onclick="handleCreateDir()">
|
||||
<span class="glyphicon glyphicon-plus" aria-hidden="true"></span> New Folder
|
||||
</label>
|
||||
<label class="btn btn-default" for="fileElem">
|
||||
<span class="glyphicon glyphicon-cloud-upload" aria-hidden="true"></span> Upload
|
||||
</label>
|
||||
</div>
|
||||
<ol class="breadcrumb">
|
||||
{{ range $entry := .Breadcrumbs }}
|
||||
<a href="{{ printpath $entry.Link }}">
|
||||
<li><a href="{{ printpath $entry.Link }}">
|
||||
{{ $entry.Name }}
|
||||
</a>
|
||||
</li></a>
|
||||
{{ end }}
|
||||
<label class="button" for="fileElem">Upload</label>
|
||||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -61,13 +109,18 @@
|
|||
<form class="upload-form">
|
||||
<input type="file" id="fileElem" multiple onchange="handleFiles(this.files)">
|
||||
|
||||
<table width="90%">
|
||||
{{if .EmptyFolder}}
|
||||
<div class="row add-files">
|
||||
+
|
||||
</div>
|
||||
{{else}}
|
||||
<table width="100%" class="table table-hover">
|
||||
{{$path := .Path }}
|
||||
{{ range $entry_index, $entry := .Entries }}
|
||||
<tr>
|
||||
<td>
|
||||
{{if $entry.IsDirectory}}
|
||||
<img src="/seaweedfsstatic/images/folder.gif" width="20" height="23">
|
||||
<span class="glyphicon glyphicon-folder-open" aria-hidden="true"></span>
|
||||
<a href="{{ printpath $path "/" $entry.Name "/"}}" >
|
||||
{{ $entry.Name }}
|
||||
</a>
|
||||
|
|
@ -89,13 +142,29 @@
|
|||
{{ $entry.Size | humanizeBytes }}
|
||||
{{end}}
|
||||
</td>
|
||||
<td nowrap>
|
||||
<td align="right" nowrap>
|
||||
{{ $entry.Timestamp.Format "2006-01-02 15:04" }}
|
||||
</td>
|
||||
<td style="width:75px">
|
||||
<div class="btn-group btn-group-xs pull-right operations" role="group">
|
||||
<label class="btn" onclick="handleRename('{{ $entry.Name }}', '{{ printpath $path "/" }}')">
|
||||
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{if $entry.IsDirectory}}
|
||||
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
|
||||
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{else}}
|
||||
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name }}')">
|
||||
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{end}}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
|
||||
</table>
|
||||
{{end}}
|
||||
</form>
|
||||
</div>
|
||||
|
||||
|
|
@ -109,65 +178,177 @@
|
|||
|
||||
<br/>
|
||||
<br/>
|
||||
|
||||
<div id="progress-area" class="footer" style="display: none;">
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
<script type="text/javascript">
|
||||
// ************************ Drag and drop ***************** //
|
||||
let dropArea = document.getElementById("drop-area")
|
||||
let dropArea = document.getElementById("drop-area");
|
||||
let progressArea = document.getElementById("progress-area");
|
||||
|
||||
// Prevent default drag behaviors
|
||||
;['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
|
||||
dropArea.addEventListener(eventName, preventDefaults, false)
|
||||
document.body.addEventListener(eventName, preventDefaults, false)
|
||||
})
|
||||
dropArea.addEventListener(eventName, preventDefaults, false);
|
||||
document.body.addEventListener(eventName, preventDefaults, false);
|
||||
});
|
||||
|
||||
// Highlight drop area when item is dragged over it
|
||||
;['dragenter', 'dragover'].forEach(eventName => {
|
||||
dropArea.addEventListener(eventName, highlight, false)
|
||||
})
|
||||
dropArea.addEventListener(eventName, highlight, false);
|
||||
});
|
||||
|
||||
;['dragleave', 'drop'].forEach(eventName => {
|
||||
dropArea.addEventListener(eventName, unhighlight, false)
|
||||
})
|
||||
dropArea.addEventListener(eventName, unhighlight, false);
|
||||
});
|
||||
|
||||
// Handle dropped files
|
||||
dropArea.addEventListener('drop', handleDrop, false)
|
||||
dropArea.addEventListener('drop', handleDrop, false);
|
||||
|
||||
function preventDefaults(e) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
}
|
||||
|
||||
function highlight(e) {
|
||||
dropArea.classList.add('highlight')
|
||||
dropArea.classList.add('highlight');
|
||||
}
|
||||
|
||||
function unhighlight(e) {
|
||||
dropArea.classList.remove('highlight')
|
||||
dropArea.classList.remove('highlight');
|
||||
}
|
||||
|
||||
function handleDrop(e) {
|
||||
var dt = e.dataTransfer
|
||||
var files = dt.files
|
||||
var dt = e.dataTransfer;
|
||||
var files = dt.files;
|
||||
|
||||
handleFiles(files)
|
||||
handleFiles(files);
|
||||
}
|
||||
|
||||
var uploadList = {};
|
||||
|
||||
function handleFiles(files) {
|
||||
files = [...files]
|
||||
files.forEach(uploadFile)
|
||||
window.location.reload()
|
||||
files = [...files];
|
||||
files.forEach(startUpload);
|
||||
renderProgress();
|
||||
files.forEach(uploadFile);
|
||||
}
|
||||
|
||||
function startUpload(file, i) {
|
||||
uploadList[file.name] = {'name': file.name, 'percent': 0, 'finish': false};
|
||||
}
|
||||
|
||||
function renderProgress() {
|
||||
var values = Object.values(uploadList);
|
||||
var html = '<table class="table">\n<tr><th>Uploading</th><\/tr>\n';
|
||||
for (let i of values) {
|
||||
var progressBarClass = 'progress-bar-striped active';
|
||||
if (i.percent >= 100) {
|
||||
progressBarClass = 'progress-bar-success';
|
||||
}
|
||||
html += '<tr>\n<td>\n';
|
||||
html += '<div class="progress" style="margin-bottom: 2px;">\n';
|
||||
html += '<div class="progress-bar ' + progressBarClass + '" role="progressbar" aria-valuenow="' + '100" aria-valuemin="0" aria-valuemax="100" style="width:' + i.percent + '%;">';
|
||||
html += '<span style="margin-right: 10px;">' + i.name + '</span>' + i.percent + '%<\/div>';
|
||||
html += '<\/div>\n<\/td>\n<\/tr>\n';
|
||||
}
|
||||
html += '<\/table>\n';
|
||||
progressArea.innerHTML = html;
|
||||
if (values.length > 0) {
|
||||
progressArea.attributes.style.value = '';
|
||||
}
|
||||
}
|
||||
|
||||
function reportProgress(file, percent) {
|
||||
var item = uploadList[file]
|
||||
item.percent = percent;
|
||||
renderProgress();
|
||||
}
|
||||
|
||||
function finishUpload(file) {
|
||||
uploadList[file]['finish'] = true;
|
||||
renderProgress();
|
||||
var allFinish = true;
|
||||
for (let i of Object.values(uploadList)) {
|
||||
if (!i.finish) {
|
||||
allFinish = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (allFinish) {
|
||||
console.log('All Finish');
|
||||
window.location.reload();
|
||||
}
|
||||
}
|
||||
|
||||
function uploadFile(file, i) {
|
||||
var url = window.location.href
|
||||
var xhr = new XMLHttpRequest()
|
||||
var formData = new FormData()
|
||||
xhr.open('POST', url, false)
|
||||
var url = window.location.href;
|
||||
var xhr = new XMLHttpRequest();
|
||||
var fileName = file.name;
|
||||
xhr.upload.addEventListener('progress', function(e) {
|
||||
if (e.lengthComputable) {
|
||||
var percent = Math.ceil((e.loaded / e.total) * 100);
|
||||
reportProgress(fileName, percent)
|
||||
}
|
||||
});
|
||||
xhr.upload.addEventListener('loadend', function(e) {
|
||||
finishUpload(fileName);
|
||||
});
|
||||
var formData = new FormData();
|
||||
xhr.open('POST', url, true);
|
||||
formData.append('file', file);
|
||||
xhr.send(formData);
|
||||
}
|
||||
|
||||
formData.append('file', file)
|
||||
xhr.send(formData)
|
||||
function handleCreateDir() {
|
||||
var dirName = prompt('Folder Name:', '');
|
||||
dirName = dirName.trim();
|
||||
if (dirName == null || dirName == '') {
|
||||
return;
|
||||
}
|
||||
var baseUrl = window.location.href;
|
||||
if (!baseUrl.endsWith('/')) {
|
||||
baseUrl += '/';
|
||||
}
|
||||
var url = baseUrl + dirName;
|
||||
if (!url.endsWith('/')) {
|
||||
url += '/';
|
||||
}
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', url, false);
|
||||
xhr.setRequestHeader('Content-Type', '');
|
||||
xhr.send();
|
||||
window.location.reload();
|
||||
}
|
||||
|
||||
function handleRename(originName, basePath) {
|
||||
var newName = prompt('New Name:', originName);
|
||||
if (newName == null || newName == '') {
|
||||
return;
|
||||
}
|
||||
var url = basePath + newName;
|
||||
var originPath = basePath + originName;
|
||||
url += '?mv.from=' + originPath;
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', url, false);
|
||||
xhr.setRequestHeader('Content-Type', '');
|
||||
xhr.send();
|
||||
window.location.reload();
|
||||
}
|
||||
|
||||
function handleDelete(path) {
|
||||
if (!confirm('Are you sure to delete ' + path + '?')) {
|
||||
return;
|
||||
}
|
||||
var url = path;
|
||||
if (url.endsWith('/')) {
|
||||
url += '?recursive=true';
|
||||
}
|
||||
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('DELETE', url, false);
|
||||
xhr.send();
|
||||
window.location.reload();
|
||||
}
|
||||
</script>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -113,6 +113,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
|||
}
|
||||
|
||||
if len(heartbeat.Volumes) > 0 || heartbeat.HasNoVolumes {
|
||||
dcName, rackName := ms.Topo.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
|
||||
ms.Topo.DataNodeRegistration(dcName, rackName, dn)
|
||||
|
||||
// process heartbeat.Volumes
|
||||
stats.MasterReceivedHeartbeatCounter.WithLabelValues("Volumes").Inc()
|
||||
newVolumes, deletedVolumes := ms.Topo.SyncDataNodeRegistration(heartbeat.Volumes, dn)
|
||||
|
|
@ -133,13 +136,13 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
|||
ms.Topo.IncrementalSyncDataNodeEcShards(heartbeat.NewEcShards, heartbeat.DeletedEcShards, dn)
|
||||
|
||||
for _, s := range heartbeat.NewEcShards {
|
||||
message.NewVids = append(message.NewVids, s.Id)
|
||||
message.NewEcVids = append(message.NewEcVids, s.Id)
|
||||
}
|
||||
for _, s := range heartbeat.DeletedEcShards {
|
||||
if dn.HasVolumesById(needle.VolumeId(s.Id)) {
|
||||
if dn.HasEcShards(needle.VolumeId(s.Id)) {
|
||||
continue
|
||||
}
|
||||
message.DeletedVids = append(message.DeletedVids, s.Id)
|
||||
message.DeletedEcVids = append(message.DeletedEcVids, s.Id)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -151,17 +154,17 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
|||
|
||||
// broadcast the ec vid changes to master clients
|
||||
for _, s := range newShards {
|
||||
message.NewVids = append(message.NewVids, uint32(s.VolumeId))
|
||||
message.NewEcVids = append(message.NewEcVids, uint32(s.VolumeId))
|
||||
}
|
||||
for _, s := range deletedShards {
|
||||
if dn.HasVolumesById(s.VolumeId) {
|
||||
continue
|
||||
}
|
||||
message.DeletedVids = append(message.DeletedVids, uint32(s.VolumeId))
|
||||
message.DeletedEcVids = append(message.DeletedEcVids, uint32(s.VolumeId))
|
||||
}
|
||||
|
||||
}
|
||||
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
|
||||
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 || len(message.NewEcVids) > 0 || len(message.DeletedEcVids) > 0 {
|
||||
ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ package weed_server
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/cluster"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -142,3 +146,41 @@ func (ms *MasterServer) ReleaseAdminToken(ctx context.Context, req *master_pb.Re
|
|||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (resp *master_pb.PingResponse, pingErr error) {
|
||||
resp = &master_pb.PingResponse{
|
||||
StartTimeNs: time.Now().UnixNano(),
|
||||
}
|
||||
if req.TargetType == cluster.FilerType {
|
||||
pingErr = pb.WithFilerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if req.TargetType == cluster.VolumeServerType {
|
||||
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if req.TargetType == cluster.MasterType {
|
||||
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client master_pb.SeaweedClient) error {
|
||||
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
|
||||
if pingResp != nil {
|
||||
resp.RemoteTimeNs = pingResp.StartTimeNs
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if pingErr != nil {
|
||||
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
|
||||
}
|
||||
resp.StopTimeNs = time.Now().UnixNano()
|
||||
return
|
||||
}
|
||||
|
|
|
|||
66
weed/server/master_grpc_server_raft.go
Normal file
66
weed/server/master_grpc_server_raft.go
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/cluster"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/hashicorp/raft"
|
||||
)
|
||||
|
||||
func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_pb.RaftListClusterServersRequest) (*master_pb.RaftListClusterServersResponse, error) {
|
||||
resp := &master_pb.RaftListClusterServersResponse{}
|
||||
|
||||
servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
|
||||
|
||||
for _, server := range servers {
|
||||
resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{
|
||||
Id: string(server.ID),
|
||||
Address: string(server.Address),
|
||||
Suffrage: server.Suffrage.String(),
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (ms *MasterServer) RaftAddServer(ctx context.Context, req *master_pb.RaftAddServerRequest) (*master_pb.RaftAddServerResponse, error) {
|
||||
resp := &master_pb.RaftAddServerResponse{}
|
||||
if ms.Topo.HashicorpRaft.State() != raft.Leader {
|
||||
return nil, fmt.Errorf("raft add server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
|
||||
}
|
||||
|
||||
var idxFuture raft.IndexFuture
|
||||
if req.Voter {
|
||||
idxFuture = ms.Topo.HashicorpRaft.AddVoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
|
||||
} else {
|
||||
idxFuture = ms.Topo.HashicorpRaft.AddNonvoter(raft.ServerID(req.Id), raft.ServerAddress(req.Address), 0, 0)
|
||||
}
|
||||
|
||||
if err := idxFuture.Error(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (ms *MasterServer) RaftRemoveServer(ctx context.Context, req *master_pb.RaftRemoveServerRequest) (*master_pb.RaftRemoveServerResponse, error) {
|
||||
resp := &master_pb.RaftRemoveServerResponse{}
|
||||
|
||||
if ms.Topo.HashicorpRaft.State() != raft.Leader {
|
||||
return nil, fmt.Errorf("raft remove server %s failed: %s is no current leader", req.Id, ms.Topo.HashicorpRaft.String())
|
||||
}
|
||||
|
||||
if !req.Force {
|
||||
ms.clientChansLock.RLock()
|
||||
_, ok := ms.clientChans[fmt.Sprintf("%s@%s", cluster.MasterType, req.Id)]
|
||||
ms.clientChansLock.RUnlock()
|
||||
if ok {
|
||||
return resp, fmt.Errorf("raft remove server %s failed: client connection to master exists", req.Id)
|
||||
}
|
||||
}
|
||||
|
||||
idxFuture := ms.Topo.HashicorpRaft.RemoveServer(raft.ServerID(req.Id), 0, 0)
|
||||
if err := idxFuture.Error(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
|
@ -268,7 +268,7 @@ func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumV
|
|||
|
||||
resp := &master_pb.VacuumVolumeResponse{}
|
||||
|
||||
ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.preallocateSize)
|
||||
ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), req.VolumeId, req.Collection, ms.preallocateSize)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"net/http"
|
||||
|
|
@ -17,6 +18,7 @@ import (
|
|||
|
||||
"github.com/chrislusf/raft"
|
||||
"github.com/gorilla/mux"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
|
@ -30,8 +32,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
SequencerType = "master.sequencer.type"
|
||||
SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
|
||||
SequencerType = "master.sequencer.type"
|
||||
SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
|
||||
RaftServerRemovalTime = 72 * time.Minute
|
||||
)
|
||||
|
||||
type MasterOption struct {
|
||||
|
|
@ -62,6 +65,9 @@ type MasterServer struct {
|
|||
|
||||
boundedLeaderChan chan int
|
||||
|
||||
onPeerUpdatDoneCn chan string
|
||||
onPeerUpdatDoneCnExist bool
|
||||
|
||||
// notifying clients
|
||||
clientChansLock sync.RWMutex
|
||||
clientChans map[string]chan *master_pb.KeepConnectedResponse
|
||||
|
|
@ -75,7 +81,7 @@ type MasterServer struct {
|
|||
Cluster *cluster.Cluster
|
||||
}
|
||||
|
||||
func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {
|
||||
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
|
||||
|
||||
v := util.GetViper()
|
||||
signingKey := v.GetString("jwt.signing.key")
|
||||
|
|
@ -112,6 +118,9 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddre
|
|||
Cluster: cluster.NewCluster(),
|
||||
}
|
||||
ms.boundedLeaderChan = make(chan int, 16)
|
||||
ms.onPeerUpdatDoneCn = make(chan string)
|
||||
|
||||
ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
|
||||
|
||||
seq := ms.createSequencer(option)
|
||||
if nil == seq {
|
||||
|
|
@ -160,19 +169,41 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddre
|
|||
}
|
||||
|
||||
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
|
||||
ms.Topo.RaftServer = raftServer.raftServer
|
||||
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
|
||||
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
|
||||
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
|
||||
if ms.Topo.RaftServer.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
|
||||
}
|
||||
})
|
||||
var raftServerName string
|
||||
if raftServer.raftServer != nil {
|
||||
ms.Topo.RaftServer = raftServer.raftServer
|
||||
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
|
||||
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
|
||||
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
|
||||
if ms.Topo.RaftServer.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
|
||||
}
|
||||
})
|
||||
raftServerName = ms.Topo.RaftServer.Name()
|
||||
} else if raftServer.RaftHashicorp != nil {
|
||||
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
|
||||
leaderCh := raftServer.RaftHashicorp.LeaderCh()
|
||||
prevLeader := ms.Topo.HashicorpRaft.Leader()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case isLeader := <-leaderCh:
|
||||
leader := ms.Topo.HashicorpRaft.Leader()
|
||||
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
|
||||
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
|
||||
prevLeader = leader
|
||||
}
|
||||
}
|
||||
}()
|
||||
raftServerName = ms.Topo.HashicorpRaft.String()
|
||||
}
|
||||
if ms.Topo.IsLeader() {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!")
|
||||
glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
|
||||
} else {
|
||||
if ms.Topo.RaftServer.Leader() != "" {
|
||||
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
|
||||
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -181,31 +212,38 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
|
|||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if ms.Topo.IsLeader() {
|
||||
f(w, r)
|
||||
} else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
|
||||
ms.boundedLeaderChan <- 1
|
||||
defer func() { <-ms.boundedLeaderChan }()
|
||||
targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader())
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError,
|
||||
fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err))
|
||||
return
|
||||
}
|
||||
glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader())
|
||||
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
|
||||
director := proxy.Director
|
||||
proxy.Director = func(req *http.Request) {
|
||||
actualHost, err := security.GetActualRemoteHost(req)
|
||||
if err == nil {
|
||||
req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
|
||||
}
|
||||
director(req)
|
||||
}
|
||||
proxy.Transport = util.Transport
|
||||
proxy.ServeHTTP(w, r)
|
||||
} else {
|
||||
// handle requests locally
|
||||
f(w, r)
|
||||
return
|
||||
}
|
||||
var raftServerLeader string
|
||||
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
|
||||
raftServerLeader = ms.Topo.RaftServer.Leader()
|
||||
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
|
||||
raftServerLeader = string(ms.Topo.HashicorpRaft.Leader())
|
||||
}
|
||||
if raftServerLeader == "" {
|
||||
f(w, r)
|
||||
return
|
||||
}
|
||||
ms.boundedLeaderChan <- 1
|
||||
defer func() { <-ms.boundedLeaderChan }()
|
||||
targetUrl, err := url.Parse("http://" + raftServerLeader)
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError,
|
||||
fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
|
||||
return
|
||||
}
|
||||
glog.V(4).Infoln("proxying to leader", raftServerLeader)
|
||||
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
|
||||
director := proxy.Director
|
||||
proxy.Director = func(req *http.Request) {
|
||||
actualHost, err := security.GetActualRemoteHost(req)
|
||||
if err == nil {
|
||||
req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
|
||||
}
|
||||
director(req)
|
||||
}
|
||||
proxy.Transport = util.Transport
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -301,3 +339,57 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
|
|||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
|
||||
if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
|
||||
return
|
||||
}
|
||||
glog.V(4).Infof("OnPeerUpdate: %+v", update)
|
||||
|
||||
peerAddress := pb.ServerAddress(update.Address)
|
||||
peerName := string(peerAddress)
|
||||
isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
|
||||
if update.IsAdd {
|
||||
if isLeader {
|
||||
raftServerFound := false
|
||||
for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
|
||||
if string(server.ID) == peerName {
|
||||
raftServerFound = true
|
||||
}
|
||||
}
|
||||
if !raftServerFound {
|
||||
glog.V(0).Infof("adding new raft server: %s", peerName)
|
||||
ms.Topo.HashicorpRaft.AddVoter(
|
||||
hashicorpRaft.ServerID(peerName),
|
||||
hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
|
||||
}
|
||||
}
|
||||
if ms.onPeerUpdatDoneCnExist {
|
||||
ms.onPeerUpdatDoneCn <- peerName
|
||||
}
|
||||
} else if isLeader {
|
||||
go func(peerName string) {
|
||||
for {
|
||||
select {
|
||||
case <-time.After(RaftServerRemovalTime):
|
||||
err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
|
||||
_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
|
||||
Id: peerName,
|
||||
Force: false,
|
||||
})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
|
||||
}
|
||||
return
|
||||
case peerDone := <-ms.onPeerUpdatDoneCn:
|
||||
if peerName == peerDone {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}(peerName)
|
||||
ms.onPeerUpdatDoneCnExist = true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
}
|
||||
// glog.Infoln("garbageThreshold =", gcThreshold)
|
||||
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
|
||||
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, 0, "", ms.preallocateSize)
|
||||
ms.dirStatusHandler(w, r)
|
||||
}
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue