mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'master' into gentle_vacuum
# Conflicts: # weed/pb/messaging_pb/messaging.pb.go # weed/pb/messaging_pb/messaging_grpc.pb.go # weed/pb/s3_pb/s3.pb.go # weed/pb/volume_server_pb/volume_server.pb.go # weed/server/volume_grpc_vacuum.go
This commit is contained in:
Konstantin Lebedev
commit
3c75479e2b
6
.github/ISSUE_TEMPLATE.md
vendored
6
.github/ISSUE_TEMPLATE.md
vendored
|
|
@ -9,12 +9,12 @@ assignees: ''
|
|||
|
||||
Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
|
||||
Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
|
||||
Please ask questions in https://github.com/chrislusf/seaweedfs/discussions
|
||||
Please ask questions in https://github.com/seaweedfs/seaweedfs/discussions
|
||||
|
||||
example of a good issue report:
|
||||
https://github.com/chrislusf/seaweedfs/issues/1005
|
||||
https://github.com/seaweedfs/seaweedfs/issues/1005
|
||||
example of a bad issue report:
|
||||
https://github.com/chrislusf/seaweedfs/issues/1008
|
||||
https://github.com/seaweedfs/seaweedfs/issues/1008
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
|
|
|||
8
.github/workflows/binaries_dev.yml
vendored
8
.github/workflows/binaries_dev.yml
vendored
|
|
@ -53,7 +53,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-large-disk
|
||||
|
|
@ -68,7 +68,7 @@ jobs:
|
|||
release_tag: dev
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-normal-disk
|
||||
|
|
@ -102,7 +102,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-large-disk
|
||||
|
|
@ -117,7 +117,7 @@ jobs:
|
|||
release_tag: dev
|
||||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed-normal-disk
|
||||
|
|
|
|||
4
.github/workflows/binaries_release0.yml
vendored
4
.github/workflows/binaries_release0.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
|
|||
4
.github/workflows/binaries_release1.yml
vendored
4
.github/workflows/binaries_release1.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
|
|||
4
.github/workflows/binaries_release2.yml
vendored
4
.github/workflows/binaries_release2.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
|
|||
4
.github/workflows/binaries_release3.yml
vendored
4
.github/workflows/binaries_release3.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
|
|||
4
.github/workflows/binaries_release4.yml
vendored
4
.github/workflows/binaries_release4.yml
vendored
|
|
@ -39,7 +39,7 @@ jobs:
|
|||
build_flags: -tags elastic,ydb,gocdk,tikv
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
# build_flags: -tags 5BytesOffset # optional, default is
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
@ -53,7 +53,7 @@ jobs:
|
|||
overwrite: true
|
||||
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
|
||||
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||
# Where to run `go build .`
|
||||
project_path: weed
|
||||
binary_name: weed
|
||||
|
|
|
|||
69
README.md
69
README.md
|
|
@ -3,19 +3,19 @@
|
|||
|
||||
[](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
|
||||
[](https://twitter.com/intent/follow?screen_name=seaweedfs)
|
||||
[](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
|
||||
[](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
[](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
|
||||
[](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<h2 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h2>
|
||||
|
||||
SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
|
||||
possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
|
||||
possible entirely thanks to the support of these awesome [backers](https://github.com/seaweedfs/seaweedfs/blob/master/backers.md).
|
||||
If you'd like to grow SeaweedFS even stronger, please consider joining our
|
||||
<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
|
||||
|
||||
|
|
@ -36,14 +36,14 @@ Your support will be really appreciated by me and other supporters!
|
|||
|
||||
---
|
||||
|
||||
- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
|
||||
- [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
|
||||
- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
|
||||
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
|
||||
- [SeaweedFS on Telegram](https://t.me/Seaweedfs)
|
||||
- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/)
|
||||
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
|
||||
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
|
||||
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
|
||||
- [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
- [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
|
||||
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
|
||||
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ Table of Contents
|
|||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
|
@ -159,25 +159,25 @@ Faster and Cheaper than direct cloud storage!
|
|||
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
|
||||
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
|
||||
|
||||
[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
|
||||
[SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files
|
||||
[Mount]: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount
|
||||
[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
|
||||
[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Async-Replication-to-Cloud
|
||||
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
|
||||
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
|
||||
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
|
||||
[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
|
||||
[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
|
||||
[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
|
||||
[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
|
||||
[VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
|
||||
[Filer]: https://github.com/seaweedfs/seaweedfs/wiki/Directories-and-Files
|
||||
[SuperLargeFiles]: https://github.com/seaweedfs/seaweedfs/wiki/Data-Structure-for-Large-Files
|
||||
[Mount]: https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount
|
||||
[AmazonS3API]: https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API
|
||||
[BackupToCloud]: https://github.com/seaweedfs/seaweedfs/wiki/Async-Replication-to-Cloud
|
||||
[Hadoop]: https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System
|
||||
[WebDAV]: https://github.com/seaweedfs/seaweedfs/wiki/WebDAV
|
||||
[ErasureCoding]: https://github.com/seaweedfs/seaweedfs/wiki/Erasure-coding-for-warm-storage
|
||||
[TieredStorage]: https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage
|
||||
[CloudTier]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Tier
|
||||
[FilerDataEncryption]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption
|
||||
[FilerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores
|
||||
[VolumeServerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Store-file-with-a-Time-To-Live
|
||||
[SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver
|
||||
[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
|
||||
[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
|
||||
[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
[ActiveActiveAsyncReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
|
||||
[FilerStoreReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication
|
||||
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
|
@ -300,7 +300,7 @@ The replication parameter options are:
|
|||
|
||||
More details about replication can be found [on the wiki][Replication].
|
||||
|
||||
[Replication]: https://github.com/chrislusf/seaweedfs/wiki/Replication
|
||||
[Replication]: https://github.com/seaweedfs/seaweedfs/wiki/Replication
|
||||
|
||||
You can also set the default replication strategy when starting the master server.
|
||||
|
||||
|
|
@ -325,10 +325,10 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
|
|||
* [Chunking large files][feat-3]
|
||||
* [Collection as a Simple Name Space][feat-4]
|
||||
|
||||
[feat-1]: https://github.com/chrislusf/seaweedfs/wiki/Failover-Master-Server
|
||||
[feat-2]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#insert-with-your-own-keys
|
||||
[feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
|
||||
[feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
|
||||
[feat-1]: https://github.com/seaweedfs/seaweedfs/wiki/Failover-Master-Server
|
||||
[feat-2]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#insert-with-your-own-keys
|
||||
[feat-3]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#upload-large-files
|
||||
[feat-4]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
|
|
@ -442,7 +442,7 @@ MooseFS Master Server keeps all meta data in memory. Same issue as HDFS namenode
|
|||
|
||||
### Compared to Ceph ###
|
||||
|
||||
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
|
||||
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/seaweedfs/seaweedfs/issues/120)
|
||||
|
||||
SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
|
||||
|
||||
|
|
@ -502,7 +502,7 @@ make sure to define your $GOPATH
|
|||
|
||||
Step 2: checkout this repo:
|
||||
```bash
|
||||
git clone https://github.com/chrislusf/seaweedfs.git
|
||||
git clone https://github.com/seaweedfs/seaweedfs.git
|
||||
```
|
||||
Step 3: download, compile, and install the project by executing the following command
|
||||
|
||||
|
|
@ -601,4 +601,3 @@ The text of this page is available for modification and reuse under the terms of
|
|||
## Stargazers over time
|
||||
|
||||
[](https://starchart.cc/chrislusf/seaweedfs)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
FROM gcc:11 as builder
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y golang-src \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse # for weed mount
|
||||
|
||||
# volume server gprc port
|
||||
|
|
@ -41,4 +41,4 @@ WORKDIR /data
|
|||
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
FROM golang:1.18-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
ARG TAGS
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse # for weed mount
|
||||
|
||||
# volume server gprc port
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ FROM golang:1.18-buster as builder
|
|||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v7.2.2
|
||||
ENV ROCKSDB_VERSION v7.4.4
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
|
|
@ -17,11 +17,11 @@ ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4
|
|||
|
||||
# build SeaweedFS
|
||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
|
||||
RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
|
||||
|
|
@ -29,8 +29,8 @@ FROM alpine AS final
|
|||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
RUN apk add fuse snappy gflags
|
||||
|
||||
# volume server gprc port
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ gen: dev
|
|||
|
||||
binary:
|
||||
export SWCOMMIT=$(shell git rev-parse --short HEAD)
|
||||
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
||||
export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
||||
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
|
||||
|
||||
build: binary
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
```bash
|
||||
|
||||
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
|
|
@ -15,7 +15,7 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
|||
|
||||
```bash
|
||||
|
||||
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
|||
## Local Development
|
||||
|
||||
```bash
|
||||
cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
|
||||
cd $GOPATH/src/github.com/seaweedfs/seaweedfs/docker
|
||||
make
|
||||
```
|
||||
|
||||
|
|
@ -48,4 +48,4 @@ docker buildx stop $BUILDER
|
|||
```
|
||||
mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1
|
||||
mc admin trace --all --verbose local
|
||||
```
|
||||
```
|
||||
|
|
|
|||
19
go.mod
19
go.mod
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/chrislusf/seaweedfs
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.18
|
||||
|
||||
|
|
@ -8,15 +8,11 @@ require (
|
|||
cloud.google.com/go/storage v1.24.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/Shopify/sarama v1.35.0
|
||||
github.com/aws/aws-sdk-go v1.44.56
|
||||
github.com/aws/aws-sdk-go v1.44.65
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
|
|
@ -88,8 +84,8 @@ require (
|
|||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.8.2 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
|
|
@ -126,7 +122,7 @@ require (
|
|||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.87.0
|
||||
google.golang.org/api v0.90.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
|
||||
google.golang.org/grpc v1.48.0
|
||||
|
|
@ -139,21 +135,22 @@ require (
|
|||
modernc.org/mathutil v1.4.1 // indirect
|
||||
modernc.org/memory v1.1.1 // indirect
|
||||
modernc.org/opt v0.1.1 // indirect
|
||||
modernc.org/sqlite v1.17.3
|
||||
modernc.org/sqlite v1.18.0
|
||||
modernc.org/strutil v1.1.2
|
||||
modernc.org/token v1.0.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.2.0
|
||||
github.com/arangodb/go-driver v1.3.2
|
||||
github.com/arangodb/go-driver v1.3.3
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers v2.0.6+incompatible
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.9
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
|
||||
github.com/tikv/client-go/v2 v2.0.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
|
|
|
|||
33
go.sum
33
go.sum
|
|
@ -132,8 +132,6 @@ github.com/Jille/raft-grpc-transport v1.2.0 h1:W/YSPz8IsirEyomjKmDog5Xk71o9+l4Kh
|
|||
github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
|
||||
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/Shopify/sarama v1.35.0 h1:opEGHcK8s5OpQF99wW0D4ol7A3qUpfSFigrDXnWmOcs=
|
||||
github.com/Shopify/sarama v1.35.0/go.mod h1:n8obse6Cz5NjjXjKwR1JeYr7CkQn4KG+HENJ8n/T9oQ=
|
||||
github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64=
|
||||
|
|
@ -145,8 +143,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
|
|||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/arangodb/go-driver v1.3.2 h1:07cmMqPqEl+/MlosjFtVVakEbkPqBSUvw9S9/atX0+4=
|
||||
github.com/arangodb/go-driver v1.3.2/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ=
|
||||
github.com/arangodb/go-driver v1.3.3 h1:UEV74zLYvvAvQgQgNHORdvaADtDX2uTVZH+1H7WAFAQ=
|
||||
github.com/arangodb/go-driver v1.3.3/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ=
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
|
|
@ -157,8 +155,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
|
|||
github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
|
||||
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.44.56 h1:bT+lExwagH7djxb6InKUVkEKGPAj5aAPnV85/m1fKro=
|
||||
github.com/aws/aws-sdk-go v1.44.56/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.44.65 h1:G+kuQ0+kcg8ltLZqju3OA9NDtGsGuSDrNWaXwgYFEH8=
|
||||
github.com/aws/aws-sdk-go v1.44.65/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
|
||||
|
|
@ -206,13 +204,10 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
|
|||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
|
||||
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
|
||||
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
|
||||
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
|
|
@ -411,6 +406,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
|
|||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
|
||||
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
||||
github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw=
|
||||
github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
|
|
@ -677,7 +674,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
|
|||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
|
||||
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
|
|
@ -839,8 +835,6 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
|
|||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
|
||||
github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
|
||||
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
|
||||
|
|
@ -920,8 +914,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mo
|
|||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5 h1:iKZJjrOorFYGB+2g8+WmjjnFxudEnfE70ucAXcNYxPQ=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5/go.mod h1:bsYHcRuCdelVeIwNsJicIz60flewCwp8Kg9gfwMPR/Q=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0 h1:b5+zAp4tRt2e442Qv8slIK7UiNsagacXpGQaOr/hQok=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0/go.mod h1:bsYHcRuCdelVeIwNsJicIz60flewCwp8Kg9gfwMPR/Q=
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
|
||||
|
|
@ -1402,8 +1396,8 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
|
|||
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
|
||||
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
|
||||
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
|
||||
google.golang.org/api v0.87.0 h1:pUQVF/F+X7Tl1lo4LJoJf5BOpjtmINU80p9XpYTU2p4=
|
||||
google.golang.org/api v0.87.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
|
||||
google.golang.org/api v0.90.0 h1:WMnUWAvihIClUYFNeFA69VTuR3duKS3IalMGDQcLvq8=
|
||||
google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -1612,7 +1606,6 @@ modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo=
|
|||
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
|
||||
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA=
|
||||
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
|
||||
|
|
@ -1632,18 +1625,16 @@ modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU=
|
|||
modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
|
||||
modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
|
||||
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sqlite v1.17.3 h1:iE+coC5g17LtByDYDWKpR6m2Z9022YrSh3bumwOnIrI=
|
||||
modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k=
|
||||
modernc.org/sqlite v1.18.0 h1:ef66qJSgKeyLyrF4kQ2RHw/Ue3V89fyFNbGL073aDjI=
|
||||
modernc.org/sqlite v1.18.0/go.mod h1:B9fRWZacNxJBHoCJZQr1R54zhVn3fjfl0aszflrTSxY=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.2 h1:iFBDH6j1Z0bN/Q9udJnnFoFpENA4252qe/7/5woE5MI=
|
||||
modernc.org/strutil v1.1.2/go.mod h1:OYajnUAcI/MX+XD/Wx7v1bbdvcQSvxgtb0gC+u3d3eg=
|
||||
modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao=
|
||||
modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
|
||||
modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
|
||||
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM=
|
||||
modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
|
||||
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.18"
|
||||
version: "3.18"
|
||||
appVersion: "3.19"
|
||||
version: "3.19"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
## SEAWEEDFS - helm chart (2.x)
|
||||
# SEAWEEDFS - helm chart (2.x)
|
||||
|
||||
### info:
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
|
||||
|
|
@ -9,12 +9,26 @@ and backup/HA memsql can provide.
|
|||
with ENV.
|
||||
* cert config exists and can be enabled, but not been tested.
|
||||
|
||||
### prerequisites
|
||||
kubernetes node have labels which help to define which node(Host) will run which pod.
|
||||
## Prerequisites
|
||||
### Database
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS filemeta (
|
||||
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
name VARCHAR(1000) BINARY COMMENT 'directory or file name',
|
||||
directory TEXT BINARY COMMENT 'full path to parent directory',
|
||||
meta LONGBLOB,
|
||||
PRIMARY KEY (dirhash, name)
|
||||
) DEFAULT CHARSET=utf8;
|
||||
```
|
||||
|
||||
s3/filer/master needs the label **sw-backend=true**
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
volume need the label **sw-volume=true**
|
||||
### Node Labels
|
||||
Kubernetes node have labels which help to define which node(Host) will run which pod:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
|
|
@ -29,7 +43,7 @@ please set/update the corresponding affinity rule in values.yaml to an empty one
|
|||
|
||||
```affinity: ""```
|
||||
|
||||
### PVC - storage class ###
|
||||
## PVC - storage class ###
|
||||
|
||||
on the volume stateful set added support for K8S PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
|
|
@ -38,7 +52,7 @@ https://github.com/rancher/local-path-provisioner
|
|||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
### current instances config (AIO):
|
||||
## current instances config (AIO):
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
you can update the replicas count for each node type in values.yaml,
|
||||
|
|
|
|||
BIN
note/SeaweedMQ_Architecture.png
Normal file
BIN
note/SeaweedMQ_Architecture.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 136 KiB |
|
|
@ -2,7 +2,7 @@ syntax = "proto3";
|
|||
|
||||
package filer_pb;
|
||||
|
||||
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/filer_pb";
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
|
||||
option java_package = "seaweedfs.client";
|
||||
option java_outer_classname = "FilerProto";
|
||||
|
||||
|
|
@ -60,12 +60,6 @@ service SeaweedFiler {
|
|||
rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
|
||||
}
|
||||
|
||||
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
|
||||
}
|
||||
|
||||
rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) {
|
||||
}
|
||||
|
||||
rpc KvGet (KvGetRequest) returns (KvGetResponse) {
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ parts:
|
|||
plugin: go
|
||||
# Snapcraft will look in this location for the source of the application
|
||||
source: .
|
||||
go-importpath: github.com/chrislusf/seaweedfs
|
||||
go-importpath: github.com/seaweedfs/seaweedfs
|
||||
go:
|
||||
# Defines the version of golang which will be bootstrapped into the snap
|
||||
source-tag: go1.14
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ import (
|
|||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -10,15 +10,15 @@ import (
|
|||
"math"
|
||||
"os"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/idx"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,12 +8,12 @@ import (
|
|||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"google.golang.org/grpc"
|
||||
"strconv"
|
||||
"time"
|
||||
|
|
@ -51,7 +51,7 @@ func main() {
|
|||
}
|
||||
|
||||
func startGenerateMetadata() {
|
||||
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
|
||||
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
for i := 0; i < *n; i++ {
|
||||
name := fmt.Sprintf("file%d", i)
|
||||
|
|
@ -77,7 +77,7 @@ func startGenerateMetadata() {
|
|||
|
||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
|
|
|||
|
|
@ -6,11 +6,11 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -3,16 +3,16 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"log"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -7,11 +7,11 @@ import (
|
|||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/idx"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@ import (
|
|||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
logdataFile = flag.String("logdata", "", "log data file saved under "+ filer.SystemLogDir)
|
||||
logdataFile = flag.String("logdata", "", "log data file saved under "+filer.SystemLogDir)
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ import (
|
|||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ import (
|
|||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
util2 "github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
util2 "github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/tools/godoc/util"
|
||||
)
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ func main() {
|
|||
sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
|
||||
}
|
||||
|
||||
err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
|
||||
err := operation.TailVolume(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
|
||||
if n.Size == 0 {
|
||||
println("-", n.String())
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
package cluster
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -15,88 +15,156 @@ const (
|
|||
BrokerType = "broker"
|
||||
)
|
||||
|
||||
type FilerGroup string
|
||||
type Filers struct {
|
||||
filers map[pb.ServerAddress]*ClusterNode
|
||||
leaders *Leaders
|
||||
}
|
||||
type FilerGroupName string
|
||||
type DataCenter string
|
||||
type Rack string
|
||||
|
||||
type Leaders struct {
|
||||
leaders [3]pb.ServerAddress
|
||||
}
|
||||
|
||||
type ClusterNode struct {
|
||||
Address pb.ServerAddress
|
||||
Version string
|
||||
counter int
|
||||
CreatedTs time.Time
|
||||
Address pb.ServerAddress
|
||||
Version string
|
||||
counter int
|
||||
CreatedTs time.Time
|
||||
DataCenter DataCenter
|
||||
Rack Rack
|
||||
}
|
||||
type GroupMembers struct {
|
||||
members map[pb.ServerAddress]*ClusterNode
|
||||
leaders *Leaders
|
||||
}
|
||||
type ClusterNodeGroups struct {
|
||||
groupMembers map[FilerGroupName]*GroupMembers
|
||||
sync.RWMutex
|
||||
}
|
||||
type Cluster struct {
|
||||
filerGroups *ClusterNodeGroups
|
||||
brokerGroups *ClusterNodeGroups
|
||||
}
|
||||
|
||||
type Cluster struct {
|
||||
filerGroup2filers map[FilerGroup]*Filers
|
||||
filersLock sync.RWMutex
|
||||
brokers map[pb.ServerAddress]*ClusterNode
|
||||
brokersLock sync.RWMutex
|
||||
func newClusterNodeGroups() *ClusterNodeGroups {
|
||||
return &ClusterNodeGroups{
|
||||
groupMembers: map[FilerGroupName]*GroupMembers{},
|
||||
}
|
||||
}
|
||||
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
|
||||
members, found := g.groupMembers[filerGroup]
|
||||
if !found && createIfNotFound {
|
||||
members = &GroupMembers{
|
||||
members: make(map[pb.ServerAddress]*ClusterNode),
|
||||
leaders: &Leaders{},
|
||||
}
|
||||
g.groupMembers[filerGroup] = members
|
||||
}
|
||||
return members
|
||||
}
|
||||
|
||||
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
|
||||
if existingNode, found := m.members[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
t := &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
}
|
||||
m.members[address] = t
|
||||
return t
|
||||
}
|
||||
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
|
||||
if existingNode, found := m.members[address]; !found {
|
||||
return false
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(m.members, address)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, true)
|
||||
if t := m.addMember(dataCenter, rack, address, version); t != nil {
|
||||
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
if m.removeMember(address) {
|
||||
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
for _, node := range m.members {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return
|
||||
}
|
||||
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return false
|
||||
}
|
||||
return m.leaders.isOneLeader(address)
|
||||
}
|
||||
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return m.leaders.GetLeaders()
|
||||
}
|
||||
|
||||
func NewCluster() *Cluster {
|
||||
return &Cluster{
|
||||
filerGroup2filers: make(map[FilerGroup]*Filers),
|
||||
brokers: make(map[pb.ServerAddress]*ClusterNode),
|
||||
filerGroups: newClusterNodeGroups(),
|
||||
brokerGroups: newClusterNodeGroups(),
|
||||
}
|
||||
}
|
||||
|
||||
func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers {
|
||||
filers, found := cluster.filerGroup2filers[filerGroup]
|
||||
if !found && createIfNotFound {
|
||||
filers = &Filers{
|
||||
filers: make(map[pb.ServerAddress]*ClusterNode),
|
||||
leaders: &Leaders{},
|
||||
}
|
||||
cluster.filerGroup2filers[filerGroup] = filers
|
||||
}
|
||||
return filers
|
||||
}
|
||||
|
||||
func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroup(ns)
|
||||
func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.Lock()
|
||||
defer cluster.filersLock.Unlock()
|
||||
filers := cluster.getFilers(filerGroup, true)
|
||||
if existingNode, found := filers.filers[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
filers.filers[address] = &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
}
|
||||
return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
|
||||
return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.Lock()
|
||||
defer cluster.brokersLock.Unlock()
|
||||
if existingNode, found := cluster.brokers[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
cluster.brokers[address] = &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
}
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroupName(ns)
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
|
|
@ -112,44 +180,12 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAdd
|
|||
}
|
||||
|
||||
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
|
||||
filerGroup := FilerGroup(ns)
|
||||
filerGroup := FilerGroupName(ns)
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.Lock()
|
||||
defer cluster.filersLock.Unlock()
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return nil
|
||||
}
|
||||
if existingNode, found := filers.filers[address]; !found {
|
||||
return nil
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(filers.filers, address)
|
||||
return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
|
||||
}
|
||||
}
|
||||
return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.Lock()
|
||||
defer cluster.brokersLock.Unlock()
|
||||
if existingNode, found := cluster.brokers[address]; !found {
|
||||
return nil
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(cluster.brokers, address)
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
|
|
@ -164,40 +200,42 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb
|
|||
return nil
|
||||
}
|
||||
|
||||
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) {
|
||||
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
cluster.filersLock.RLock()
|
||||
defer cluster.filersLock.RUnlock()
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return
|
||||
}
|
||||
for _, node := range filers.filers {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return cluster.filerGroups.ListClusterNode(filerGroup)
|
||||
case BrokerType:
|
||||
cluster.brokersLock.RLock()
|
||||
defer cluster.brokersLock.RUnlock()
|
||||
for _, node := range cluster.brokers {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
return cluster.brokerGroups.ListClusterNode(filerGroup)
|
||||
case MasterType:
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool {
|
||||
filers := cluster.getFilers(filerGroup, false)
|
||||
if filers == nil {
|
||||
return false
|
||||
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case MasterType:
|
||||
}
|
||||
return filers.leaders.isOneLeader(address)
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.IsOneLeader(filerGroup, address)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
|
||||
case MasterType:
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
if isAdd {
|
||||
if filers.leaders.addLeaderIfVacant(address) {
|
||||
if m.leaders.addLeaderIfVacant(address) {
|
||||
// has added the address as one leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
|
|
@ -220,7 +258,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
|||
})
|
||||
}
|
||||
} else {
|
||||
if filers.leaders.removeLeaderIfExists(address) {
|
||||
if m.leaders.removeLeaderIfExists(address) {
|
||||
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
|
|
@ -236,8 +274,8 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
|||
var shortestDuration int64 = math.MaxInt64
|
||||
now := time.Now()
|
||||
var candidateAddress pb.ServerAddress
|
||||
for _, node := range filers.filers {
|
||||
if filers.leaders.isOneLeader(node.Address) {
|
||||
for _, node := range m.members {
|
||||
if m.leaders.isOneLeader(node.Address) {
|
||||
continue
|
||||
}
|
||||
duration := now.Sub(node.CreatedTs).Nanoseconds()
|
||||
|
|
@ -247,7 +285,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
|
|||
}
|
||||
}
|
||||
if candidateAddress != "" {
|
||||
filers.leaders.addLeaderIfVacant(candidateAddress)
|
||||
m.leaders.addLeaderIfVacant(candidateAddress)
|
||||
// added a new leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package cluster
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
|
@ -11,29 +11,29 @@ import (
|
|||
func TestClusterAddRemoveNodes(t *testing.T) {
|
||||
c := NewCluster()
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:1"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:2"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:3"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:4"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45")
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:5"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:6"), "23.45")
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
|
|
@ -41,7 +41,7 @@ func TestClusterAddRemoveNodes(t *testing.T) {
|
|||
pb.ServerAddress("111:6"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getFilers("", false).leaders.GetLeaders())
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
|
|
@ -56,7 +56,7 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
|
|||
go func(i int) {
|
||||
defer wg.Done()
|
||||
address := strconv.Itoa(i)
|
||||
c.AddClusterNode("", "filer", pb.ServerAddress(address), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress(address), "23.45")
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
|
|
|||
34
weed/cluster/master_client.go
Normal file
34
weed/cluster/master_client.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
|
||||
|
||||
if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
|
||||
ClientType: clientType,
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
|
||||
glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
|
||||
for _, node := range resp.ClusterNodes {
|
||||
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
|
||||
NodeType: FilerType,
|
||||
Address: node.Address,
|
||||
IsLeader: node.IsLeader,
|
||||
IsAdd: true,
|
||||
CreatedAtNs: node.CreatedAtNs,
|
||||
})
|
||||
}
|
||||
return err
|
||||
}); grpcErr != nil {
|
||||
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -2,9 +2,9 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
"github.com/posener/complete"
|
||||
completeinstall "github.com/posener/complete/cmd/install"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,15 +2,15 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package command
|
|||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand"
|
||||
|
|
@ -16,11 +16,11 @@ import (
|
|||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
type BenchmarkOptions struct {
|
||||
|
|
@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
|
|||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
|
||||
go b.masterClient.KeepConnectedToMaster()
|
||||
b.masterClient.WaitUntilConnected()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var Commands = []*Command{
|
||||
|
|
@ -28,12 +28,12 @@ var Commands = []*Command{
|
|||
cmdFilerSynchronize,
|
||||
cmdFix,
|
||||
cmdFuse,
|
||||
cmdIam,
|
||||
cmdMaster,
|
||||
cmdMasterFollower,
|
||||
cmdMount,
|
||||
cmdMqBroker,
|
||||
cmdS3,
|
||||
cmdIam,
|
||||
cmdMsgBroker,
|
||||
cmdScaffold,
|
||||
cmdServer,
|
||||
cmdShell,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ import (
|
|||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -13,13 +13,13 @@ import (
|
|||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ import (
|
|||
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
|
|||
|
|
@ -3,17 +3,17 @@ package command
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -14,15 +14,15 @@ import (
|
|||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -3,17 +3,17 @@ package command
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -2,16 +2,16 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/golang/protobuf/jsonpb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ package command
|
|||
|
||||
import (
|
||||
"context"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
elastic "github.com/olivere/elastic/v7"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@ package command
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"os"
|
||||
"time"
|
||||
|
|
|
|||
|
|
@ -2,16 +2,16 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/remote_storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"math"
|
||||
"math/rand"
|
||||
"path/filepath"
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"time"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,15 +7,15 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/remote_storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ import (
|
|||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/sink"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/sub"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sub"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -4,17 +4,17 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/sink"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
statsCollect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"google.golang.org/grpc"
|
||||
"os"
|
||||
"strings"
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/iamapi"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/iamapi"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,34 +3,34 @@ package command
|
|||
import (
|
||||
_ "net/http/pprof"
|
||||
|
||||
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/azure"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/gcs"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/s3"
|
||||
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/azuresink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/b2sink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/gcssink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/s3sink"
|
||||
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/hbase"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/tikv"
|
||||
_ "github.com/chrislusf/seaweedfs/weed/filer/ydb"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/elastic/v7"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/etcd"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/hbase"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb2"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb3"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis2"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis3"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/sqlite"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/tikv"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/filer/ydb"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,28 +2,30 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"golang.org/x/exp/slices"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/raft/protobuf"
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/backend"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -174,11 +176,12 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
|||
} else {
|
||||
raftServer, err = weed_server.NewRaftServer(raftServerOption)
|
||||
if raftServer == nil {
|
||||
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
|
||||
glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
|
||||
}
|
||||
}
|
||||
ms.SetRaftServer(raftServer)
|
||||
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
||||
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
|
||||
if *m.raftHashicorp {
|
||||
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/reflection"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,16 +6,16 @@ package command
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/mount"
|
||||
"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/chrislusf/seaweedfs/weed/mount/unmount"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/unmount"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net"
|
||||
"net/http"
|
||||
|
|
@ -26,8 +26,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
func runMount(cmd *Command, args []string) bool {
|
||||
|
|
|
|||
94
weed/command/mq_broker.go
Normal file
94
weed/command/mq_broker.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/broker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
mqBrokerStandaloneOptions MessageQueueBrokerOptions
|
||||
)
|
||||
|
||||
type MessageQueueBrokerOptions struct {
|
||||
masters map[string]pb.ServerAddress
|
||||
mastersString *string
|
||||
filerGroup *string
|
||||
ip *string
|
||||
port *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdMqBroker.Run = runMqBroker // break init cycle
|
||||
mqBrokerStandaloneOptions.mastersString = cmdMqBroker.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
|
||||
mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
|
||||
mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
|
||||
mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
|
||||
mqBrokerStandaloneOptions.cpuprofile = cmdMqBroker.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
mqBrokerStandaloneOptions.memprofile = cmdMqBroker.Flag.String("memprofile", "", "memory profile output file")
|
||||
}
|
||||
|
||||
var cmdMqBroker = &Command{
|
||||
UsageLine: "mq.broker [-port=17777] [-master=<ip:port>]",
|
||||
Short: "<WIP> start a message queue broker",
|
||||
Long: `start a message queue broker
|
||||
|
||||
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
|
||||
The brokers are stateless. To scale up, just add more brokers.
|
||||
|
||||
`,
|
||||
}
|
||||
|
||||
func runMqBroker(cmd *Command, args []string) bool {
|
||||
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
mqBrokerStandaloneOptions.masters = pb.ServerAddresses(*mqBrokerStandaloneOptions.mastersString).ToAddressMap()
|
||||
|
||||
return mqBrokerStandaloneOptions.startQueueServer()
|
||||
|
||||
}
|
||||
|
||||
func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
|
||||
|
||||
grace.SetupProfiling(*mqBrokerStandaloneOptions.cpuprofile, *mqBrokerStandaloneOptions.memprofile)
|
||||
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
|
||||
|
||||
qs, err := broker.NewMessageBroker(&broker.MessageQueueBrokerOption{
|
||||
Masters: mqBrokerOpt.masters,
|
||||
FilerGroup: *mqBrokerOpt.filerGroup,
|
||||
DataCenter: *mqBrokerOpt.dataCenter,
|
||||
Rack: *mqBrokerOpt.rack,
|
||||
DefaultReplication: "",
|
||||
MaxMB: 0,
|
||||
Ip: *mqBrokerOpt.ip,
|
||||
Port: *mqBrokerOpt.port,
|
||||
}, grpcDialOption)
|
||||
|
||||
// start grpc listener
|
||||
grpcL, _, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
|
||||
mq_pb.RegisterSeaweedMessagingServer(grpcS, qs)
|
||||
reflection.Register(grpcS)
|
||||
grpcS.Serve(grpcL)
|
||||
|
||||
return true
|
||||
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/messaging/broker"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
messageBrokerStandaloneOptions MessageBrokerOptions
|
||||
)
|
||||
|
||||
type MessageBrokerOptions struct {
|
||||
filer *string
|
||||
ip *string
|
||||
port *int
|
||||
cpuprofile *string
|
||||
memprofile *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdMsgBroker.Run = runMsgBroker // break init cycle
|
||||
messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address")
|
||||
messageBrokerStandaloneOptions.ip = cmdMsgBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
|
||||
messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "broker gRPC listen port")
|
||||
messageBrokerStandaloneOptions.cpuprofile = cmdMsgBroker.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
messageBrokerStandaloneOptions.memprofile = cmdMsgBroker.Flag.String("memprofile", "", "memory profile output file")
|
||||
}
|
||||
|
||||
var cmdMsgBroker = &Command{
|
||||
UsageLine: "msgBroker [-port=17777] [-filer=<ip:port>]",
|
||||
Short: "start a message queue broker",
|
||||
Long: `start a message queue broker
|
||||
|
||||
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
|
||||
The brokers are stateless. To scale up, just add more brokers.
|
||||
|
||||
`,
|
||||
}
|
||||
|
||||
func runMsgBroker(cmd *Command, args []string) bool {
|
||||
|
||||
util.LoadConfiguration("security", false)
|
||||
|
||||
return messageBrokerStandaloneOptions.startQueueServer()
|
||||
|
||||
}
|
||||
|
||||
func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
|
||||
|
||||
grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile)
|
||||
|
||||
filerAddress := pb.ServerAddress(*msgBrokerOpt.filer)
|
||||
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
|
||||
cipher := false
|
||||
|
||||
for {
|
||||
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
|
||||
}
|
||||
cipher = resp.Cipher
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
|
||||
time.Sleep(time.Second)
|
||||
} else {
|
||||
glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{
|
||||
Filers: []pb.ServerAddress{filerAddress},
|
||||
DefaultReplication: "",
|
||||
MaxMB: 0,
|
||||
Ip: *msgBrokerOpt.ip,
|
||||
Port: *msgBrokerOpt.port,
|
||||
Cipher: cipher,
|
||||
}, grpcDialOption)
|
||||
|
||||
// start grpc listener
|
||||
grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
|
||||
messaging_pb.RegisterSeaweedMessagingServer(grpcS, qs)
|
||||
reflection.Register(grpcS)
|
||||
grpcS.Serve(grpcL)
|
||||
|
||||
return true
|
||||
|
||||
}
|
||||
|
|
@ -3,22 +3,22 @@ package command
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/command/scaffold"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command/scaffold"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
type ServerOptions struct {
|
||||
|
|
@ -24,13 +24,13 @@ type ServerOptions struct {
|
|||
}
|
||||
|
||||
var (
|
||||
serverOptions ServerOptions
|
||||
masterOptions MasterOptions
|
||||
filerOptions FilerOptions
|
||||
s3Options S3Options
|
||||
iamOptions IamOptions
|
||||
webdavOptions WebDavOption
|
||||
msgBrokerOptions MessageBrokerOptions
|
||||
serverOptions ServerOptions
|
||||
masterOptions MasterOptions
|
||||
filerOptions FilerOptions
|
||||
s3Options S3Options
|
||||
iamOptions IamOptions
|
||||
webdavOptions WebDavOption
|
||||
mqBrokerOptions MessageQueueBrokerOptions
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -74,7 +74,7 @@ var (
|
|||
isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
|
||||
isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
|
||||
isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")
|
||||
isStartingMqBroker = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
|
||||
|
||||
serverWhiteList []string
|
||||
|
||||
|
|
@ -155,7 +155,7 @@ func init() {
|
|||
webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
|
||||
webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
|
||||
|
||||
msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")
|
||||
mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ func runServer(cmd *Command, args []string) bool {
|
|||
if *isStartingWebDav {
|
||||
*isStartingFiler = true
|
||||
}
|
||||
if *isStartingMsgBroker {
|
||||
if *isStartingMqBroker {
|
||||
*isStartingFiler = true
|
||||
}
|
||||
|
||||
|
|
@ -208,7 +208,9 @@ func runServer(cmd *Command, args []string) bool {
|
|||
serverOptions.v.idleConnectionTimeout = serverTimeout
|
||||
serverOptions.v.dataCenter = serverDataCenter
|
||||
serverOptions.v.rack = serverRack
|
||||
msgBrokerOptions.ip = serverIp
|
||||
mqBrokerOptions.ip = serverIp
|
||||
mqBrokerOptions.masters = filerOptions.masters
|
||||
mqBrokerOptions.filerGroup = filerOptions.filerGroup
|
||||
|
||||
// serverOptions.v.pulseSeconds = pulseSeconds
|
||||
// masterOptions.pulseSeconds = pulseSeconds
|
||||
|
|
@ -217,6 +219,8 @@ func runServer(cmd *Command, args []string) bool {
|
|||
|
||||
filerOptions.dataCenter = serverDataCenter
|
||||
filerOptions.rack = serverRack
|
||||
mqBrokerOptions.dataCenter = serverDataCenter
|
||||
mqBrokerOptions.rack = serverRack
|
||||
filerOptions.disableHttp = serverDisableHttp
|
||||
masterOptions.disableHttp = serverDisableHttp
|
||||
|
||||
|
|
@ -224,7 +228,7 @@ func runServer(cmd *Command, args []string) bool {
|
|||
s3Options.filer = &filerAddress
|
||||
iamOptions.filer = &filerAddress
|
||||
webdavOptions.filer = &filerAddress
|
||||
msgBrokerOptions.filer = &filerAddress
|
||||
mqBrokerOptions.filerGroup = filerOptions.filerGroup
|
||||
|
||||
go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
|
||||
|
||||
|
|
@ -276,10 +280,10 @@ func runServer(cmd *Command, args []string) bool {
|
|||
}()
|
||||
}
|
||||
|
||||
if *isStartingMsgBroker {
|
||||
if *isStartingMqBroker {
|
||||
go func() {
|
||||
time.Sleep(2 * time.Second)
|
||||
msgBrokerOptions.startQueueServer()
|
||||
mqBrokerOptions.startQueueServer()
|
||||
}()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/shell"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/shell"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
)
|
||||
|
||||
|
|
@ -76,8 +76,8 @@ func init() {
|
|||
|
||||
var cmdUpdate = &Command{
|
||||
UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]",
|
||||
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
|
||||
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
|
||||
Short: "get latest or specific version from https://github.com/seaweedfs/seaweedfs",
|
||||
Long: `get latest or specific version from https://github.com/seaweedfs/seaweedfs`,
|
||||
}
|
||||
|
||||
func runUpdate(cmd *Command, args []string) bool {
|
||||
|
|
@ -118,7 +118,7 @@ func runUpdate(cmd *Command, args []string) bool {
|
|||
|
||||
func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
|
||||
currentVersion := util.VERSION_NUMBER
|
||||
rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs")
|
||||
rel, err := GitHubLatestRelease(ctx, ver, "seaweedfs", "seaweedfs")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,15 +4,15 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"google.golang.org/grpc"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var cmdVersion = &Command{
|
||||
|
|
|
|||
|
|
@ -10,25 +10,25 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/httpdown"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
|
||||
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func TestXYZ(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@ import (
|
|||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
"github.com/chrislusf/seaweedfs/weed/server"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/server"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ import (
|
|||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import (
|
|||
"database/sql"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,10 +11,10 @@ import (
|
|||
|
||||
"github.com/arangodb/go-driver"
|
||||
"github.com/arangodb/go-driver/http"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import (
|
|||
"github.com/arangodb/go-driver"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
var _ filer.BucketAware = (*ArangodbStore)(nil)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/arangodb/go-driver"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/arangodb/go-driver"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
//convert a string into arango-key safe hex bytes hash
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ import (
|
|||
"github.com/gocql/gocql"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import (
|
|||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/gocql/gocql"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@ import (
|
|||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
weed_util "github.com/chrislusf/seaweedfs/weed/util"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
elastic "github.com/olivere/elastic/v7"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -7,11 +7,11 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
elastic "github.com/olivere/elastic/v7"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func (store *ElasticStore) KvDelete(ctx context.Context, key []byte) (err error) {
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ import (
|
|||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
type Attr struct {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ import (
|
|||
|
||||
"go.etcd.io/etcd/client/v3"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
weed_util "github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package etcd
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
)
|
||||
|
||||
func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package etcd
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/filer/store_test"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer/store_test"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,13 +10,13 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func TestDoMaybeManifestize(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -3,12 +3,12 @@ package filer
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/exp/slices"
|
||||
"math"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
|
||||
|
|
@ -114,9 +114,12 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
|
|||
fileIds := make(map[string]bool)
|
||||
for _, interval := range bs {
|
||||
fileIds[interval.GetFileIdString()] = true
|
||||
fileIds[interval.GetSourceFileId()] = true
|
||||
}
|
||||
for _, chunk := range as {
|
||||
if _, found := fileIds[chunk.GetSourceFileId()]; !found {
|
||||
_, sourceFileIdFound := fileIds[chunk.GetSourceFileId()]
|
||||
_, fileIdFound := fileIds[chunk.GetFileId()]
|
||||
if !sourceFileIdFound && !fileIdFound {
|
||||
delta = append(delta, chunk)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,52 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/exp/slices"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func TestDoMinusChunks(t *testing.T) {
|
||||
// https://github.com/seaweedfs/seaweedfs/issues/3328
|
||||
|
||||
// clusterA and clusterB using filer.sync to sync file: hello.txt
|
||||
// clusterA append a new line and then clusterB also append a new line
|
||||
// clusterA append a new line again
|
||||
chunksInA := []*filer_pb.FileChunk{
|
||||
{Offset: 0, Size: 3, FileId: "11", Mtime: 100},
|
||||
{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", Mtime: 200},
|
||||
{Offset: 6, Size: 3, FileId: "33", Mtime: 300},
|
||||
}
|
||||
chunksInB := []*filer_pb.FileChunk{
|
||||
{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", Mtime: 100},
|
||||
{Offset: 3, Size: 3, FileId: "2", Mtime: 200},
|
||||
{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", Mtime: 300},
|
||||
}
|
||||
|
||||
// clusterB using command "echo 'content' > hello.txt" to overwrite file
|
||||
// clusterA will receive two evenNotification, need to empty the whole file content first and add new content
|
||||
// the first one is oldEntry is chunksInB and newEntry is empty fileChunks
|
||||
firstOldEntry := chunksInB
|
||||
var firstNewEntry []*filer_pb.FileChunk
|
||||
|
||||
// clusterA received the first one event, gonna empty the whole chunk, according the code in filer_sink 194
|
||||
// we can get the deleted chunks and newChunks
|
||||
firstDeletedChunks := DoMinusChunks(firstOldEntry, firstNewEntry)
|
||||
log.Println("first deleted chunks:", firstDeletedChunks)
|
||||
//firstNewEntry := DoMinusChunks(firstNewEntry, firstOldEntry)
|
||||
|
||||
// clusterA need to delete all chunks in firstDeletedChunks
|
||||
emptiedChunksInA := DoMinusChunksBySourceFileId(chunksInA, firstDeletedChunks)
|
||||
// chunksInA supposed to be empty by minus the deletedChunks but it just delete the chunk which sync from clusterB
|
||||
log.Println("clusterA synced empty chunks event result:", emptiedChunksInA)
|
||||
// clusterB emptied it's chunks and clusterA must sync the change and empty chunks too
|
||||
assert.Equalf(t, firstNewEntry, emptiedChunksInA, "empty")
|
||||
}
|
||||
|
||||
func TestCompactFileChunksRealCase(t *testing.T) {
|
||||
|
||||
chunks := []*filer_pb.FileChunk{
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package filer
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func TestCompactFileChunks(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ package filer
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/cluster"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
|
@ -13,11 +13,11 @@ import (
|
|||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -52,7 +52,7 @@ type Filer struct {
|
|||
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
|
||||
filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
|
||||
f := &Filer{
|
||||
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, masters),
|
||||
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
|
||||
fileIdDeletionQueue: util.NewUnboundedQueue(),
|
||||
GrpcDialOption: grpcDialOption,
|
||||
FilerConf: NewFilerConf(),
|
||||
|
|
@ -105,28 +105,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
|
|||
}
|
||||
|
||||
func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
|
||||
|
||||
if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
|
||||
ClientType: cluster.FilerType,
|
||||
FilerGroup: f.MasterClient.FilerGroup,
|
||||
})
|
||||
|
||||
glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
|
||||
for _, node := range resp.ClusterNodes {
|
||||
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
|
||||
NodeType: cluster.FilerType,
|
||||
Address: node.Address,
|
||||
IsLeader: node.IsLeader,
|
||||
IsAdd: true,
|
||||
CreatedAtNs: node.CreatedAtNs,
|
||||
})
|
||||
}
|
||||
return err
|
||||
}); grpcErr != nil {
|
||||
glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr)
|
||||
}
|
||||
return
|
||||
return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
|
||||
}
|
||||
|
||||
func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
|
||||
|
|
|
|||
|
|
@ -4,15 +4,15 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
"io"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/golang/protobuf/jsonpb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package filer
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue