gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into gentle_vacuum

Browse source 
# Conflicts:
#	weed/pb/messaging_pb/messaging.pb.go
#	weed/pb/messaging_pb/messaging_grpc.pb.go
#	weed/pb/s3_pb/s3.pb.go
#	weed/pb/volume_server_pb/volume_server.pb.go
#	weed/server/volume_grpc_vacuum.go
This commit is contained in:
Konstantin Lebedev 2022-08-01 14:45:22 +05:00
parent c0d92f61a1 036566629a
commit 3c75479e2b
633 changed files with 6720 additions and 7638 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/ISSUE_TEMPLATE.md vendored
View file

@ -9,12 +9,12 @@ assignees: ''
Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs Report issues here. Ask questions here https://stackoverflow.com/questions/tagged/seaweedfs
Please ask questions in https://github.com/chrislusf/seaweedfs/discussions Please ask questions in https://github.com/seaweedfs/seaweedfs/discussions
example of a good issue report: example of a good issue report:
https://github.com/chrislusf/seaweedfs/issues/1005 https://github.com/seaweedfs/seaweedfs/issues/1005
example of a bad issue report: example of a bad issue report:
https://github.com/chrislusf/seaweedfs/issues/1008 https://github.com/seaweedfs/seaweedfs/issues/1008
**Describe the bug** **Describe the bug**
A clear and concise description of what the bug is. A clear and concise description of what the bug is.

8
.github/workflows/binaries_dev.yml vendored
View file

@ -53,7 +53,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed-large-disk binary_name: weed-large-disk
@ -68,7 +68,7 @@ jobs:
release_tag: dev release_tag: dev
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed-normal-disk binary_name: weed-normal-disk
@ -102,7 +102,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed-large-disk binary_name: weed-large-disk
@ -117,7 +117,7 @@ jobs:
release_tag: dev release_tag: dev
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed-normal-disk binary_name: weed-normal-disk

4
.github/workflows/binaries_release0.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is # build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed
@ -52,7 +52,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed

4
.github/workflows/binaries_release1.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is # build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed
@ -52,7 +52,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed

4
.github/workflows/binaries_release2.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is # build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed
@ -52,7 +52,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed

4
.github/workflows/binaries_release3.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is # build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed
@ -52,7 +52,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed

4
.github/workflows/binaries_release4.yml vendored
View file

@ -39,7 +39,7 @@ jobs:
build_flags: -tags elastic,ydb,gocdk,tikv build_flags: -tags elastic,ydb,gocdk,tikv
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
# build_flags: -tags 5BytesOffset # optional, default is # build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed
@ -53,7 +53,7 @@ jobs:
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0 pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed binary_name: weed

69
README.md
View file

@ -3,19 +3,19 @@
[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) [![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs) [![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml) [![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/seaweedfs/seaweedfs/actions/workflows/go.yml)
[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed) [![GoDoc](https://godoc.org/github.com/seaweedfs/seaweedfs/weed?status.svg)](https://godoc.org/github.com/seaweedfs/seaweedfs/weed)
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki) [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/seaweedfs/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/) [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf) [![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png) ![SeaweedFS Logo](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/seaweedfs.png)
<h2 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h2> <h2 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h2>
SeaweedFS is an independent Apache-licensed open source project with its ongoing development made SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). possible entirely thanks to the support of these awesome [backers](https://github.com/seaweedfs/seaweedfs/blob/master/backers.md).
If you'd like to grow SeaweedFS even stronger, please consider joining our If you'd like to grow SeaweedFS even stronger, please consider joining our
<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>. <a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
@ -36,14 +36,14 @@ Your support will be really appreciated by me and other supporters!
--- ---
- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest) - [Download Binaries for different platforms](https://github.com/seaweedfs/seaweedfs/releases/latest)
- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY) - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS) - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
- [SeaweedFS on Telegram](https://t.me/Seaweedfs) - [SeaweedFS on Telegram](https://t.me/Seaweedfs)
- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/) - [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/)
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs) - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki) - [Wiki Documentation](https://github.com/seaweedfs/seaweedfs/wiki)
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf) - [SeaweedFS White Paper](https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
- [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing) - [SeaweedFS Introduction Slides 2021.5](https://docs.google.com/presentation/d/1DcxKWlINc-HNCjhYeERkpGXXm6nTCES8mi2W5G0Z4Ts/edit?usp=sharing)
- [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction) - [SeaweedFS Introduction Slides 2019.3](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
@ -77,7 +77,7 @@ Table of Contents
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3` `docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
## Quick Start with Single Binary ## ## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it! Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
@ -159,25 +159,25 @@ Faster and Cheaper than direct cloud storage!
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/) * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator) * [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files [Filer]: https://github.com/seaweedfs/seaweedfs/wiki/Directories-and-Files
[SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files [SuperLargeFiles]: https://github.com/seaweedfs/seaweedfs/wiki/Data-Structure-for-Large-Files
[Mount]: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount [Mount]: https://github.com/seaweedfs/seaweedfs/wiki/FUSE-Mount
[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API [AmazonS3API]: https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API
[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Async-Replication-to-Cloud [BackupToCloud]: https://github.com/seaweedfs/seaweedfs/wiki/Async-Replication-to-Cloud
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System [Hadoop]: https://github.com/seaweedfs/seaweedfs/wiki/Hadoop-Compatible-File-System
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV [WebDAV]: https://github.com/seaweedfs/seaweedfs/wiki/WebDAV
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage [ErasureCoding]: https://github.com/seaweedfs/seaweedfs/wiki/Erasure-coding-for-warm-storage
[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage [TieredStorage]: https://github.com/seaweedfs/seaweedfs/wiki/Tiered-Storage
[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier [CloudTier]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Tier
[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption [FilerDataEncryption]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Data-Encryption
[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores [FilerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Stores
[VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live [VolumeServerTTL]: https://github.com/seaweedfs/seaweedfs/wiki/Store-file-with-a-Time-To-Live
[SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver [SeaweedFsCsiDriver]: https://github.com/seaweedfs/seaweedfs-csi-driver
[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization [ActiveActiveAsyncReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication [FilerStoreReplication]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication
[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store [KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture [CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage [GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[Back to TOC](#table-of-contents) [Back to TOC](#table-of-contents)
@ -300,7 +300,7 @@ The replication parameter options are:
More details about replication can be found [on the wiki][Replication]. More details about replication can be found [on the wiki][Replication].
[Replication]: https://github.com/chrislusf/seaweedfs/wiki/Replication [Replication]: https://github.com/seaweedfs/seaweedfs/wiki/Replication
You can also set the default replication strategy when starting the master server. You can also set the default replication strategy when starting the master server.
@ -325,10 +325,10 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
* [Chunking large files][feat-3] * [Chunking large files][feat-3]
* [Collection as a Simple Name Space][feat-4] * [Collection as a Simple Name Space][feat-4]
[feat-1]: https://github.com/chrislusf/seaweedfs/wiki/Failover-Master-Server [feat-1]: https://github.com/seaweedfs/seaweedfs/wiki/Failover-Master-Server
[feat-2]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#insert-with-your-own-keys [feat-2]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#insert-with-your-own-keys
[feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files [feat-3]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#upload-large-files
[feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space [feat-4]: https://github.com/seaweedfs/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
[Back to TOC](#table-of-contents) [Back to TOC](#table-of-contents)
@ -442,7 +442,7 @@ MooseFS Master Server keeps all meta data in memory. Same issue as HDFS namenode
### Compared to Ceph ### ### Compared to Ceph ###
Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120) Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/seaweedfs/seaweedfs/issues/120)
SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
@ -502,7 +502,7 @@ make sure to define your $GOPATH
Step 2: checkout this repo: Step 2: checkout this repo:
```bash ```bash
git clone https://github.com/chrislusf/seaweedfs.git git clone https://github.com/seaweedfs/seaweedfs.git
``` ```
Step 3: download, compile, and install the project by executing the following command Step 3: download, compile, and install the project by executing the following command
@ -601,4 +601,3 @@ The text of this page is available for modification and reuse under the terms of
## Stargazers over time ## Stargazers over time
[![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs) [![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs)

14
docker/Dockerfile.gccgo_build
View file

@ -1,20 +1,20 @@
FROM gcc:11 as builder FROM gcc:11 as builder
RUN mkdir -p /go/src/github.com/chrislusf/ RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master} ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& apt-get update \ && apt-get update \
&& apt-get install -y golang-src \ && apt-get install -y golang-src \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
FROM alpine AS final FROM alpine AS final
LABEL author="Chris Lu" LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
RUN apk add fuse # for weed mount RUN apk add fuse # for weed mount
# volume server gprc port # volume server gprc port
@ -41,4 +41,4 @@ WORKDIR /data
RUN chmod +x /entrypoint.sh RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"] ENTRYPOINT ["/entrypoint.sh"]

12
docker/Dockerfile.go_build
View file

@ -1,20 +1,20 @@
FROM golang:1.18-alpine as builder FROM golang:1.18-alpine as builder
RUN apk add git g++ fuse RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/ RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master} ARG BRANCH=${BRANCH:-master}
ARG TAGS ARG TAGS
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
FROM alpine AS final FROM alpine AS final
LABEL author="Chris Lu" LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
RUN apk add fuse # for weed mount RUN apk add fuse # for weed mount
# volume server gprc port # volume server gprc port

14
docker/Dockerfile.rocksdb_large
View file

@ -3,7 +3,7 @@ FROM golang:1.18-buster as builder
RUN apt-get update RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
ENV ROCKSDB_VERSION v7.2.2 ENV ROCKSDB_VERSION v7.4.4
# build RocksDB # build RocksDB
RUN cd /tmp && \ RUN cd /tmp && \
@ -17,11 +17,11 @@ ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4
# build SeaweedFS # build SeaweedFS
RUN mkdir -p /go/src/github.com/chrislusf/ RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
ARG BRANCH=${BRANCH:-master} ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}" && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
@ -29,8 +29,8 @@ FROM alpine AS final
LABEL author="Chris Lu" LABEL author="Chris Lu"
COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=builder /go/bin/weed /usr/bin/
RUN mkdir -p /etc/seaweedfs RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
RUN apk add fuse snappy gflags RUN apk add fuse snappy gflags
# volume server gprc port # volume server gprc port

2
docker/Makefile
View file

@ -6,7 +6,7 @@ gen: dev
binary: binary:
export SWCOMMIT=$(shell git rev-parse --short HEAD) export SWCOMMIT=$(shell git rev-parse --short HEAD)
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)" export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/ cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
build: binary build: binary

8
docker/README.md
View file

@ -5,7 +5,7 @@
```bash ```bash
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
docker-compose -f seaweedfs-compose.yml -p seaweedfs up docker-compose -f seaweedfs-compose.yml -p seaweedfs up
@ -15,7 +15,7 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
```bash ```bash
wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-dev-compose.yml wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
@ -24,7 +24,7 @@ docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
## Local Development ## Local Development
```bash ```bash
cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker cd $GOPATH/src/github.com/seaweedfs/seaweedfs/docker
make make
``` ```
@ -48,4 +48,4 @@ docker buildx stop $BUILDER
``` ```
mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1 mc config host add local http://127.0.0.1:9000 some_access_key1 some_secret_key1
mc admin trace --all --verbose local mc admin trace --all --verbose local
``` ```

19
go.mod
View file

@ -1,4 +1,4 @@
module github.com/chrislusf/seaweedfs module github.com/seaweedfs/seaweedfs
go 1.18 go 1.18
@ -8,15 +8,11 @@ require (
cloud.google.com/go/storage v1.24.0 cloud.google.com/go/storage v1.24.0
github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/azure-storage-blob-go v0.15.0
github.com/OneOfOne/xxhash v1.2.8
github.com/Shopify/sarama v1.35.0 github.com/Shopify/sarama v1.35.0
github.com/aws/aws-sdk-go v1.44.56 github.com/aws/aws-sdk-go v1.44.65
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0 github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/seaweedfs/raft v1.1.0
github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
@ -88,8 +84,8 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.8.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.8.2 // indirect github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect
@ -126,7 +122,7 @@ require (
golang.org/x/text v0.3.7 // indirect golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/api v0.87.0 google.golang.org/api v0.90.0
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
google.golang.org/grpc v1.48.0 google.golang.org/grpc v1.48.0
@ -139,21 +135,22 @@ require (
modernc.org/mathutil v1.4.1 // indirect modernc.org/mathutil v1.4.1 // indirect
modernc.org/memory v1.1.1 // indirect modernc.org/memory v1.1.1 // indirect
modernc.org/opt v0.1.1 // indirect modernc.org/opt v0.1.1 // indirect
modernc.org/sqlite v1.17.3 modernc.org/sqlite v1.18.0
modernc.org/strutil v1.1.2 modernc.org/strutil v1.1.2
modernc.org/token v1.0.0 // indirect modernc.org/token v1.0.0 // indirect
) )
require ( require (
github.com/Jille/raft-grpc-transport v1.2.0 github.com/Jille/raft-grpc-transport v1.2.0
github.com/arangodb/go-driver v1.3.2 github.com/arangodb/go-driver v1.3.3
github.com/fluent/fluent-logger-golang v1.9.0 github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers v2.0.6+incompatible
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17 github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
github.com/hashicorp/raft v1.3.9 github.com/hashicorp/raft v1.3.9
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
github.com/tikv/client-go/v2 v2.0.1 github.com/tikv/client-go/v2 v2.0.1
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5 github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
) )

33
go.sum
View file

@ -132,8 +132,6 @@ github.com/Jille/raft-grpc-transport v1.2.0 h1:W/YSPz8IsirEyomjKmDog5Xk71o9+l4Kh
github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc= github.com/Jille/raft-grpc-transport v1.2.0/go.mod h1:GQGUXJfjlzwA390Ox1AyVYpjCLhtGd6yqY9Sb5hpQfc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
github.com/Shopify/sarama v1.35.0 h1:opEGHcK8s5OpQF99wW0D4ol7A3qUpfSFigrDXnWmOcs= github.com/Shopify/sarama v1.35.0 h1:opEGHcK8s5OpQF99wW0D4ol7A3qUpfSFigrDXnWmOcs=
github.com/Shopify/sarama v1.35.0/go.mod h1:n8obse6Cz5NjjXjKwR1JeYr7CkQn4KG+HENJ8n/T9oQ= github.com/Shopify/sarama v1.35.0/go.mod h1:n8obse6Cz5NjjXjKwR1JeYr7CkQn4KG+HENJ8n/T9oQ=
github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64=
@ -145,8 +143,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/arangodb/go-driver v1.3.2 h1:07cmMqPqEl+/MlosjFtVVakEbkPqBSUvw9S9/atX0+4= github.com/arangodb/go-driver v1.3.3 h1:UEV74zLYvvAvQgQgNHORdvaADtDX2uTVZH+1H7WAFAQ=
github.com/arangodb/go-driver v1.3.2/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ= github.com/arangodb/go-driver v1.3.3/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ=
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
@ -157,8 +155,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go v1.44.56 h1:bT+lExwagH7djxb6InKUVkEKGPAj5aAPnV85/m1fKro= github.com/aws/aws-sdk-go v1.44.65 h1:G+kuQ0+kcg8ltLZqju3OA9NDtGsGuSDrNWaXwgYFEH8=
github.com/aws/aws-sdk-go v1.44.56/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.65/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA=
github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
@ -206,13 +204,10 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@ -411,6 +406,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw=
github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -677,7 +674,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@ -839,8 +835,6 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
@ -920,8 +914,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mo
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5 h1:iKZJjrOorFYGB+2g8+WmjjnFxudEnfE70ucAXcNYxPQ= github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0 h1:b5+zAp4tRt2e442Qv8slIK7UiNsagacXpGQaOr/hQok=
github.com/ydb-platform/ydb-go-sdk/v3 v3.29.5/go.mod h1:bsYHcRuCdelVeIwNsJicIz60flewCwp8Kg9gfwMPR/Q= github.com/ydb-platform/ydb-go-sdk/v3 v3.30.0/go.mod h1:bsYHcRuCdelVeIwNsJicIz60flewCwp8Kg9gfwMPR/Q=
github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
@ -1402,8 +1396,8 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
google.golang.org/api v0.87.0 h1:pUQVF/F+X7Tl1lo4LJoJf5BOpjtmINU80p9XpYTU2p4= google.golang.org/api v0.90.0 h1:WMnUWAvihIClUYFNeFA69VTuR3duKS3IalMGDQcLvq8=
google.golang.org/api v0.87.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1612,7 +1606,6 @@ modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo=
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA= modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA=
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
@ -1632,18 +1625,16 @@ modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU=
modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.17.3 h1:iE+coC5g17LtByDYDWKpR6m2Z9022YrSh3bumwOnIrI= modernc.org/sqlite v1.18.0 h1:ef66qJSgKeyLyrF4kQ2RHw/Ue3V89fyFNbGL073aDjI=
modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k= modernc.org/sqlite v1.18.0/go.mod h1:B9fRWZacNxJBHoCJZQr1R54zhVn3fjfl0aszflrTSxY=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
modernc.org/strutil v1.1.2 h1:iFBDH6j1Z0bN/Q9udJnnFoFpENA4252qe/7/5woE5MI= modernc.org/strutil v1.1.2 h1:iFBDH6j1Z0bN/Q9udJnnFoFpENA4252qe/7/5woE5MI=
modernc.org/strutil v1.1.2/go.mod h1:OYajnUAcI/MX+XD/Wx7v1bbdvcQSvxgtb0gC+u3d3eg= modernc.org/strutil v1.1.2/go.mod h1:OYajnUAcI/MX+XD/Wx7v1bbdvcQSvxgtb0gC+u3d3eg=
modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao= modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao=
modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM=
modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

4
k8s/helm_charts2/Chart.yaml
View file

@ -1,5 +1,5 @@
apiVersion: v1 apiVersion: v1
description: SeaweedFS description: SeaweedFS
name: seaweedfs name: seaweedfs
appVersion: "3.18" appVersion: "3.19"
version: "3.18" version: "3.19"

30
k8s/helm_charts2/README.md
View file

@ -1,6 +1,6 @@
## SEAWEEDFS - helm chart (2.x) # SEAWEEDFS - helm chart (2.x)
### info: ## Info:
* master/filer/volume are stateful sets with anti-affinity on the hostname, * master/filer/volume are stateful sets with anti-affinity on the hostname,
so your deployment will be spread/HA. so your deployment will be spread/HA.
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) * chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
@ -9,12 +9,26 @@ and backup/HA memsql can provide.
with ENV. with ENV.
* cert config exists and can be enabled, but not been tested. * cert config exists and can be enabled, but not been tested.
### prerequisites ## Prerequisites
kubernetes node have labels which help to define which node(Host) will run which pod. ### Database
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
This database should be pre-configured and initialized by running:
```sql
CREATE TABLE IF NOT EXISTS filemeta (
dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
name VARCHAR(1000) BINARY COMMENT 'directory or file name',
directory TEXT BINARY COMMENT 'full path to parent directory',
meta LONGBLOB,
PRIMARY KEY (dirhash, name)
) DEFAULT CHARSET=utf8;
```
s3/filer/master needs the label **sw-backend=true** Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
volume need the label **sw-volume=true** ### Node Labels
Kubernetes node have labels which help to define which node(Host) will run which pod:
* s3/filer/master needs the label **sw-backend=true**
* volume need the label **sw-volume=true**
to label a node to be able to run all pod types in k8s: to label a node to be able to run all pod types in k8s:
``` ```
@ -29,7 +43,7 @@ please set/update the corresponding affinity rule in values.yaml to an empty one
```affinity: ""``` ```affinity: ""```
### PVC - storage class ### ## PVC - storage class ###
on the volume stateful set added support for K8S PVC, currently example on the volume stateful set added support for K8S PVC, currently example
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s) with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
@ -38,7 +52,7 @@ https://github.com/rancher/local-path-provisioner
you can use ANY storage class you like, just update the correct storage-class you can use ANY storage class you like, just update the correct storage-class
for your deployment. for your deployment.
### current instances config (AIO): ## current instances config (AIO):
1 instance for each type (master/filer+s3/volume) 1 instance for each type (master/filer+s3/volume)
you can update the replicas count for each node type in values.yaml, you can update the replicas count for each node type in values.yaml,

BIN
note/SeaweedMQ_Architecture.png Normal file
View file

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 136 KiB

8
other/java/client/src/main/proto/filer.proto
View file

@ -2,7 +2,7 @@ syntax = "proto3";
package filer_pb; package filer_pb;
option go_package = "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"; option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb";
option java_package = "seaweedfs.client"; option java_package = "seaweedfs.client";
option java_outer_classname = "FilerProto"; option java_outer_classname = "FilerProto";
@ -60,12 +60,6 @@ service SeaweedFiler {
rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
} }
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
}
rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) {
}
rpc KvGet (KvGetRequest) returns (KvGetResponse) { rpc KvGet (KvGetRequest) returns (KvGetResponse) {
} }

2
snap/snapcraft.yaml
View file

@ -30,7 +30,7 @@ parts:
plugin: go plugin: go
# Snapcraft will look in this location for the source of the application # Snapcraft will look in this location for the source of the application
source: . source: .
go-importpath: github.com/chrislusf/seaweedfs go-importpath: github.com/seaweedfs/seaweedfs
go: go:
# Defines the version of golang which will be bootstrapped into the snap # Defines the version of golang which will be bootstrapped into the snap
source-tag: go1.14 source-tag: go1.14

8
unmaintained/change_superblock/change_superblock.go
View file

@ -7,10 +7,10 @@ import (
"path" "path"
"strconv" "strconv"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
) )
var ( var (

18
unmaintained/diff_volume_servers/diff_volume_servers.go
View file

@ -10,15 +10,15 @@ import (
"math" "math"
"os" "os"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
) )

12
unmaintained/fix_dat/fix_dat.go
View file

@ -8,12 +8,12 @@ import (
"path" "path"
"strconv" "strconv"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

10
unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
View file

@ -3,9 +3,9 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc" "google.golang.org/grpc"
"strconv" "strconv"
"time" "time"
@ -51,7 +51,7 @@ func main() {
} }
func startGenerateMetadata() { func startGenerateMetadata() {
pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error { pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
for i := 0; i < *n; i++ { for i := 0; i < *n; i++ {
name := fmt.Sprintf("file%d", i) name := fmt.Sprintf("file%d", i)
@ -77,7 +77,7 @@ func startGenerateMetadata() {
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError) tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
if tailErr != nil { if tailErr != nil {
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

10
unmaintained/remove_duplicate_fids/remove_duplicate_fids.go
View file

@ -6,11 +6,11 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
) )
var ( var (

8
unmaintained/repeated_vacuum/repeated_vacuum.go
View file

@ -3,16 +3,16 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"log" "log"
"math/rand" "math/rand"
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

10
unmaintained/see_dat/see_dat.go
View file

@ -2,13 +2,13 @@ package main
import ( import (
"flag" "flag"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
) )
var ( var (

8
unmaintained/see_idx/see_idx.go
View file

@ -7,11 +7,11 @@ import (
"path" "path"
"strconv" "strconv"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
) )
var ( var (

8
unmaintained/see_log_entry/see_log_entry.go
View file

@ -9,13 +9,13 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (
logdataFile = flag.String("logdata", "", "log data file saved under "+ filer.SystemLogDir) logdataFile = flag.String("logdata", "", "log data file saved under "+filer.SystemLogDir)
) )
func main() { func main() {

4
unmaintained/see_meta/see_meta.go
View file

@ -9,8 +9,8 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

10
unmaintained/stream_read_volume/stream_read_volume.go
View file

@ -5,11 +5,11 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"io" "io"
) )

12
unmaintained/volume_tailer/volume_tailer.go
View file

@ -2,14 +2,14 @@ package main
import ( import (
"flag" "flag"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"log" "log"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
util2 "github.com/chrislusf/seaweedfs/weed/util" util2 "github.com/seaweedfs/seaweedfs/weed/util"
"golang.org/x/tools/godoc/util" "golang.org/x/tools/godoc/util"
) )
@ -38,7 +38,7 @@ func main() {
sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano() sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
} }
err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) { err := operation.TailVolume(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
if n.Size == 0 { if n.Size == 0 {
println("-", n.String()) println("-", n.String())
return nil return nil

296
weed/cluster/cluster.go
View file

@ -1,8 +1,8 @@
package cluster package cluster
import ( import (
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"math" "math"
"sync" "sync"
"time" "time"
@ -15,88 +15,156 @@ const (
BrokerType = "broker" BrokerType = "broker"
) )
type FilerGroup string type FilerGroupName string
type Filers struct { type DataCenter string
filers map[pb.ServerAddress]*ClusterNode type Rack string
leaders *Leaders
}
type Leaders struct { type Leaders struct {
leaders [3]pb.ServerAddress leaders [3]pb.ServerAddress
} }
type ClusterNode struct { type ClusterNode struct {
Address pb.ServerAddress Address pb.ServerAddress
Version string Version string
counter int counter int
CreatedTs time.Time CreatedTs time.Time
DataCenter DataCenter
Rack Rack
}
type GroupMembers struct {
members map[pb.ServerAddress]*ClusterNode
leaders *Leaders
}
type ClusterNodeGroups struct {
groupMembers map[FilerGroupName]*GroupMembers
sync.RWMutex
}
type Cluster struct {
filerGroups *ClusterNodeGroups
brokerGroups *ClusterNodeGroups
} }
type Cluster struct { func newClusterNodeGroups() *ClusterNodeGroups {
filerGroup2filers map[FilerGroup]*Filers return &ClusterNodeGroups{
filersLock sync.RWMutex groupMembers: map[FilerGroupName]*GroupMembers{},
brokers map[pb.ServerAddress]*ClusterNode }
brokersLock sync.RWMutex }
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
members, found := g.groupMembers[filerGroup]
if !found && createIfNotFound {
members = &GroupMembers{
members: make(map[pb.ServerAddress]*ClusterNode),
leaders: &Leaders{},
}
g.groupMembers[filerGroup] = members
}
return members
}
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
if existingNode, found := m.members[address]; found {
existingNode.counter++
return nil
}
t := &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
DataCenter: dataCenter,
Rack: rack,
}
m.members[address] = t
return t
}
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
if existingNode, found := m.members[address]; !found {
return false
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(m.members, address)
return true
}
}
return false
}
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, true)
if t := m.addMember(dataCenter, rack, address, version); t != nil {
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
if m.removeMember(address) {
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
for _, node := range m.members {
nodes = append(nodes, node)
}
return
}
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return false
}
return m.leaders.isOneLeader(address)
}
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
return m.leaders.GetLeaders()
} }
func NewCluster() *Cluster { func NewCluster() *Cluster {
return &Cluster{ return &Cluster{
filerGroup2filers: make(map[FilerGroup]*Filers), filerGroups: newClusterNodeGroups(),
brokers: make(map[pb.ServerAddress]*ClusterNode), brokerGroups: newClusterNodeGroups(),
} }
} }
func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers { func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
filers, found := cluster.filerGroup2filers[filerGroup]
if !found && createIfNotFound {
filers = &Filers{
filers: make(map[pb.ServerAddress]*ClusterNode),
leaders: &Leaders{},
}
cluster.filerGroup2filers[filerGroup] = filers
}
return filers
}
func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns)
switch nodeType { switch nodeType {
case FilerType: case FilerType:
cluster.filersLock.Lock() return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
defer cluster.filersLock.Unlock()
filers := cluster.getFilers(filerGroup, true)
if existingNode, found := filers.filers[address]; found {
existingNode.counter++
return nil
}
filers.filers[address] = &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
}
return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
case BrokerType: case BrokerType:
cluster.brokersLock.Lock() return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
defer cluster.brokersLock.Unlock() }
if existingNode, found := cluster.brokers[address]; found { return nil
existingNode.counter++ }
return nil
} func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
cluster.brokers[address] = &ClusterNode{ filerGroup := FilerGroupName(ns)
Address: address, switch nodeType {
Version: version, case FilerType:
counter: 1, return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
CreatedTs: time.Now(), case BrokerType:
} return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
case MasterType: case MasterType:
return []*master_pb.KeepConnectedResponse{ return []*master_pb.KeepConnectedResponse{
{ {
@ -112,44 +180,12 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAdd
} }
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse { func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns) filerGroup := FilerGroupName(ns)
switch nodeType { switch nodeType {
case FilerType: case FilerType:
cluster.filersLock.Lock() return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
defer cluster.filersLock.Unlock()
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return nil
}
if existingNode, found := filers.filers[address]; !found {
return nil
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(filers.filers, address)
return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
}
}
case BrokerType: case BrokerType:
cluster.brokersLock.Lock() return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
defer cluster.brokersLock.Unlock()
if existingNode, found := cluster.brokers[address]; !found {
return nil
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(cluster.brokers, address)
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
}
}
case MasterType: case MasterType:
return []*master_pb.KeepConnectedResponse{ return []*master_pb.KeepConnectedResponse{
{ {
@ -164,40 +200,42 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb
return nil return nil
} }
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) { func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
switch nodeType { switch nodeType {
case FilerType: case FilerType:
cluster.filersLock.RLock() return cluster.filerGroups.ListClusterNode(filerGroup)
defer cluster.filersLock.RUnlock()
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return
}
for _, node := range filers.filers {
nodes = append(nodes, node)
}
case BrokerType: case BrokerType:
cluster.brokersLock.RLock() return cluster.brokerGroups.ListClusterNode(filerGroup)
defer cluster.brokersLock.RUnlock()
for _, node := range cluster.brokers {
nodes = append(nodes, node)
}
case MasterType: case MasterType:
} }
return return
} }
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool { func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
filers := cluster.getFilers(filerGroup, false) switch nodeType {
if filers == nil { case FilerType:
return false return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
case BrokerType:
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
case MasterType:
} }
return filers.leaders.isOneLeader(address) return
} }
func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) { func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
switch nodeType {
case FilerType:
return cluster.filerGroups.IsOneLeader(filerGroup, address)
case BrokerType:
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
case MasterType:
}
return false
}
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
if isAdd { if isAdd {
if filers.leaders.addLeaderIfVacant(address) { if m.leaders.addLeaderIfVacant(address) {
// has added the address as one leader // has added the address as one leader
result = append(result, &master_pb.KeepConnectedResponse{ result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
@ -220,7 +258,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
}) })
} }
} else { } else {
if filers.leaders.removeLeaderIfExists(address) { if m.leaders.removeLeaderIfExists(address) {
result = append(result, &master_pb.KeepConnectedResponse{ result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
@ -236,8 +274,8 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
var shortestDuration int64 = math.MaxInt64 var shortestDuration int64 = math.MaxInt64
now := time.Now() now := time.Now()
var candidateAddress pb.ServerAddress var candidateAddress pb.ServerAddress
for _, node := range filers.filers { for _, node := range m.members {
if filers.leaders.isOneLeader(node.Address) { if m.leaders.isOneLeader(node.Address) {
continue continue
} }
duration := now.Sub(node.CreatedTs).Nanoseconds() duration := now.Sub(node.CreatedTs).Nanoseconds()
@ -247,7 +285,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
} }
} }
if candidateAddress != "" { if candidateAddress != "" {
filers.leaders.addLeaderIfVacant(candidateAddress) m.leaders.addLeaderIfVacant(candidateAddress)
// added a new leader // added a new leader
result = append(result, &master_pb.KeepConnectedResponse{ result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{ ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{

24
weed/cluster/cluster_test.go
View file

@ -1,7 +1,7 @@
package cluster package cluster
import ( import (
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"strconv" "strconv"
"sync" "sync"
@ -11,29 +11,29 @@ import (
func TestClusterAddRemoveNodes(t *testing.T) { func TestClusterAddRemoveNodes(t *testing.T) {
c := NewCluster() c := NewCluster()
c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:1"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:2"), "23.45")
assert.Equal(t, []pb.ServerAddress{ assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"), pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"), pb.ServerAddress("111:2"),
}, c.getFilers("", false).leaders.GetLeaders()) }, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:3"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:4"), "23.45")
assert.Equal(t, []pb.ServerAddress{ assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"), pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"), pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"), pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders()) }, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:5"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:6"), "23.45")
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4")) c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
assert.Equal(t, []pb.ServerAddress{ assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"), pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"), pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"), pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders()) }, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
// remove oldest // remove oldest
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1")) c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
@ -41,7 +41,7 @@ func TestClusterAddRemoveNodes(t *testing.T) {
pb.ServerAddress("111:6"), pb.ServerAddress("111:6"),
pb.ServerAddress("111:2"), pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"), pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders()) }, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
// remove oldest // remove oldest
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1")) c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
@ -56,7 +56,7 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
go func(i int) { go func(i int) {
defer wg.Done() defer wg.Done()
address := strconv.Itoa(i) address := strconv.Itoa(i)
c.AddClusterNode("", "filer", pb.ServerAddress(address), "23.45") c.AddClusterNode("", "filer", "", "", pb.ServerAddress(address), "23.45")
}(i) }(i)
} }
wg.Wait() wg.Wait()

34
weed/cluster/master_client.go Normal file
View file

@ -0,0 +1,34 @@
package cluster
import (
"context"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
)
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: clientType,
FilerGroup: filerGroup,
})
glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
for _, node := range resp.ClusterNodes {
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
NodeType: FilerType,
Address: node.Address,
IsLeader: node.IsLeader,
IsAdd: true,
CreatedAtNs: node.CreatedAtNs,
})
}
return err
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
}
return
}

2
weed/command/autocomplete.go
View file

@ -2,9 +2,9 @@ package command
import ( import (
"fmt" "fmt"
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
"github.com/posener/complete" "github.com/posener/complete"
completeinstall "github.com/posener/complete/cmd/install" completeinstall "github.com/posener/complete/cmd/install"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
"runtime" "runtime"
) )

14
weed/command/backup.go
View file

@ -2,15 +2,15 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
) )
var ( var (

14
weed/command/benchmark.go
View file

@ -3,7 +3,7 @@ package command
import ( import (
"bufio" "bufio"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"io" "io"
"math" "math"
"math/rand" "math/rand"
@ -16,11 +16,11 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
) )
type BenchmarkOptions struct { type BenchmarkOptions struct {
@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile() defer pprof.StopCPUProfile()
} }
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap()) b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
go b.masterClient.KeepConnectedToMaster() go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected() b.masterClient.WaitUntilConnected()

6
weed/command/command.go
View file

@ -5,7 +5,7 @@ import (
"os" "os"
"strings" "strings"
flag "github.com/chrislusf/seaweedfs/weed/util/fla9" flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
) )
var Commands = []*Command{ var Commands = []*Command{
@ -28,12 +28,12 @@ var Commands = []*Command{
cmdFilerSynchronize, cmdFilerSynchronize,
cmdFix, cmdFix,
cmdFuse, cmdFuse,
cmdIam,
cmdMaster, cmdMaster,
cmdMasterFollower, cmdMasterFollower,
cmdMount, cmdMount,
cmdMqBroker,
cmdS3, cmdS3,
cmdIam,
cmdMsgBroker,
cmdScaffold, cmdScaffold,
cmdServer, cmdServer,
cmdShell, cmdShell,

8
weed/command/compact.go
View file

@ -1,10 +1,10 @@
package command package command
import ( import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

8
weed/command/download.go
View file

@ -10,10 +10,10 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

14
weed/command/export.go
View file

@ -13,13 +13,13 @@ import (
"text/template" "text/template"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
const ( const (

16
weed/command/filer.go
View file

@ -12,14 +12,14 @@ import (
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/chrislusf/seaweedfs/weed/server" weed_server "github.com/seaweedfs/seaweedfs/weed/server"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

10
weed/command/filer_backup.go
View file

@ -2,11 +2,11 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"strings" "strings"
"time" "time"

12
weed/command/filer_cat.go
View file

@ -3,17 +3,17 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc" "google.golang.org/grpc"
"net/url" "net/url"
"os" "os"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

18
weed/command/filer_copy.go
View file

@ -14,15 +14,15 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
) )
var ( var (

12
weed/command/filer_meta_backup.go
View file

@ -3,17 +3,17 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/spf13/viper" "github.com/spf13/viper"
"google.golang.org/grpc" "google.golang.org/grpc"
"reflect" "reflect"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

8
weed/command/filer_meta_tail.go
View file

@ -2,16 +2,16 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/jsonpb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

4
weed/command/filer_meta_tail_elastic.go
View file

@ -5,10 +5,10 @@ package command
import ( import (
"context" "context"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7" elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings" "strings"
) )

2
weed/command/filer_meta_tail_non_elastic.go
View file

@ -4,7 +4,7 @@
package command package command
import ( import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
) )
func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) { func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {

14
weed/command/filer_remote_gateway.go
View file

@ -3,13 +3,13 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb" "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"os" "os"
"time" "time"

18
weed/command/filer_remote_gateway_buckets.go
View file

@ -2,16 +2,16 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"math" "math"
"math/rand" "math/rand"
"path/filepath" "path/filepath"

12
weed/command/filer_remote_sync.go
View file

@ -2,12 +2,12 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"time" "time"
) )

16
weed/command/filer_remote_sync_dir.go
View file

@ -7,15 +7,15 @@ import (
"strings" "strings"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
) )

10
weed/command/filer_replication.go
View file

@ -4,11 +4,11 @@ import (
"context" "context"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/replication" "github.com/seaweedfs/seaweedfs/weed/replication"
"github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/sub" "github.com/seaweedfs/seaweedfs/weed/replication/sub"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

22
weed/command/filer_sync.go
View file

@ -4,17 +4,17 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication" "github.com/seaweedfs/seaweedfs/weed/replication"
"github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" "github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
"github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
statsCollect "github.com/chrislusf/seaweedfs/weed/stats" statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/seaweedfs/seaweedfs/weed/util/grace"
"google.golang.org/grpc" "google.golang.org/grpc"
"os" "os"
"strings" "strings"

14
weed/command/fix.go
View file

@ -8,13 +8,13 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

12
weed/command/iam.go
View file

@ -5,13 +5,13 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/iamapi"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iamapi"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"time" "time"
) )

56
weed/command/imports.go
View file

@ -3,34 +3,34 @@ package command
import ( import (
_ "net/http/pprof" _ "net/http/pprof"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure" _ "github.com/seaweedfs/seaweedfs/weed/remote_storage/azure"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs" _ "github.com/seaweedfs/seaweedfs/weed/remote_storage/gcs"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3" _ "github.com/seaweedfs/seaweedfs/weed/remote_storage/s3"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/azuresink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/b2sink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/gcssink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink" _ "github.com/seaweedfs/seaweedfs/weed/replication/sink/s3sink"
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb" _ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra" _ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7" _ "github.com/seaweedfs/seaweedfs/weed/filer/elastic/v7"
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd" _ "github.com/seaweedfs/seaweedfs/weed/filer/etcd"
_ "github.com/chrislusf/seaweedfs/weed/filer/hbase" _ "github.com/seaweedfs/seaweedfs/weed/filer/hbase"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb" _ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2" _ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb2"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3" _ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb3"
_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb" _ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql" _ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2" _ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2"
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres" _ "github.com/seaweedfs/seaweedfs/weed/filer/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2" _ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis" _ "github.com/seaweedfs/seaweedfs/weed/filer/redis"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2" _ "github.com/seaweedfs/seaweedfs/weed/filer/redis2"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis3" _ "github.com/seaweedfs/seaweedfs/weed/filer/redis3"
_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite" _ "github.com/seaweedfs/seaweedfs/weed/filer/sqlite"
_ "github.com/chrislusf/seaweedfs/weed/filer/tikv" _ "github.com/seaweedfs/seaweedfs/weed/filer/tikv"
_ "github.com/chrislusf/seaweedfs/weed/filer/ydb" _ "github.com/seaweedfs/seaweedfs/weed/filer/ydb"
) )

25
weed/command/master.go
View file

@ -2,28 +2,30 @@ package command
import ( import (
"fmt" "fmt"
"golang.org/x/exp/slices"
"net/http" "net/http"
"os" "os"
"path" "path"
"strings" "strings"
"time" "time"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "golang.org/x/exp/slices"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/seaweedfs/raft/protobuf" "github.com/seaweedfs/raft/protobuf"
"github.com/spf13/viper" "github.com/spf13/viper"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/util/grace" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/pb"
weed_server "github.com/chrislusf/seaweedfs/weed/server" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (
@ -174,11 +176,12 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
} else { } else {
raftServer, err = weed_server.NewRaftServer(raftServerOption) raftServer, err = weed_server.NewRaftServer(raftServerOption)
if raftServer == nil { if raftServer == nil {
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err) glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
} }
} }
ms.SetRaftServer(raftServer) ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET") r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
if *m.raftHashicorp { if *m.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET") r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
} }

12
weed/command/master_follower.go
View file

@ -7,13 +7,13 @@ import (
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
) )

22
weed/command/mount_std.go
View file

@ -6,16 +6,16 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/mount"
"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
"github.com/chrislusf/seaweedfs/weed/mount/unmount"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/hanwen/go-fuse/v2/fuse" "github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount"
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
"github.com/seaweedfs/seaweedfs/weed/mount/unmount"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"net" "net"
"net/http" "net/http"
@ -26,8 +26,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/seaweedfs/seaweedfs/weed/util/grace"
) )
func runMount(cmd *Command, args []string) bool { func runMount(cmd *Command, args []string) bool {

94
weed/command/mq_broker.go Normal file
View file

@ -0,0 +1,94 @@
package command
import (
"google.golang.org/grpc/reflection"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/broker"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (
mqBrokerStandaloneOptions MessageQueueBrokerOptions
)
type MessageQueueBrokerOptions struct {
masters map[string]pb.ServerAddress
mastersString *string
filerGroup *string
ip *string
port *int
dataCenter *string
rack *string
cpuprofile *string
memprofile *string
}
func init() {
cmdMqBroker.Run = runMqBroker // break init cycle
mqBrokerStandaloneOptions.mastersString = cmdMqBroker.Flag.String("master", "localhost:9333", "comma-separated master servers")
mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
mqBrokerStandaloneOptions.cpuprofile = cmdMqBroker.Flag.String("cpuprofile", "", "cpu profile output file")
mqBrokerStandaloneOptions.memprofile = cmdMqBroker.Flag.String("memprofile", "", "memory profile output file")
}
var cmdMqBroker = &Command{
UsageLine: "mq.broker [-port=17777] [-master=<ip:port>]",
Short: "<WIP> start a message queue broker",
Long: `start a message queue broker
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
The brokers are stateless. To scale up, just add more brokers.
`,
}
func runMqBroker(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
mqBrokerStandaloneOptions.masters = pb.ServerAddresses(*mqBrokerStandaloneOptions.mastersString).ToAddressMap()
return mqBrokerStandaloneOptions.startQueueServer()
}
func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
grace.SetupProfiling(*mqBrokerStandaloneOptions.cpuprofile, *mqBrokerStandaloneOptions.memprofile)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
qs, err := broker.NewMessageBroker(&broker.MessageQueueBrokerOption{
Masters: mqBrokerOpt.masters,
FilerGroup: *mqBrokerOpt.filerGroup,
DataCenter: *mqBrokerOpt.dataCenter,
Rack: *mqBrokerOpt.rack,
DefaultReplication: "",
MaxMB: 0,
Ip: *mqBrokerOpt.ip,
Port: *mqBrokerOpt.port,
}, grpcDialOption)
// start grpc listener
grpcL, _, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
mq_pb.RegisterSeaweedMessagingServer(grpcS, qs)
reflection.Register(grpcS)
grpcS.Serve(grpcL)
return true
}

109
weed/command/msg_broker.go
View file

@ -1,109 +0,0 @@
package command
import (
"context"
"fmt"
"time"
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/messaging/broker"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
messageBrokerStandaloneOptions MessageBrokerOptions
)
type MessageBrokerOptions struct {
filer *string
ip *string
port *int
cpuprofile *string
memprofile *string
}
func init() {
cmdMsgBroker.Run = runMsgBroker // break init cycle
messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address")
messageBrokerStandaloneOptions.ip = cmdMsgBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "broker gRPC listen port")
messageBrokerStandaloneOptions.cpuprofile = cmdMsgBroker.Flag.String("cpuprofile", "", "cpu profile output file")
messageBrokerStandaloneOptions.memprofile = cmdMsgBroker.Flag.String("memprofile", "", "memory profile output file")
}
var cmdMsgBroker = &Command{
UsageLine: "msgBroker [-port=17777] [-filer=<ip:port>]",
Short: "start a message queue broker",
Long: `start a message queue broker
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
The brokers are stateless. To scale up, just add more brokers.
`,
}
func runMsgBroker(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
return messageBrokerStandaloneOptions.startQueueServer()
}
func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile)
filerAddress := pb.ServerAddress(*msgBrokerOpt.filer)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
cipher := false
for {
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
}
cipher = resp.Cipher
return nil
})
if err != nil {
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
time.Sleep(time.Second)
} else {
glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
break
}
}
qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{
Filers: []pb.ServerAddress{filerAddress},
DefaultReplication: "",
MaxMB: 0,
Ip: *msgBrokerOpt.ip,
Port: *msgBrokerOpt.port,
Cipher: cipher,
}, grpcDialOption)
// start grpc listener
grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0)
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
messaging_pb.RegisterSeaweedMessagingServer(grpcS, qs)
reflection.Register(grpcS)
grpcS.Serve(grpcL)
return true
}

18
weed/command/s3.go
View file

@ -3,22 +3,22 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"net/http" "net/http"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb" "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/s3api" "github.com/seaweedfs/seaweedfs/weed/s3api"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

4
weed/command/scaffold.go
View file

@ -2,10 +2,10 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"path/filepath" "path/filepath"
"github.com/chrislusf/seaweedfs/weed/command/scaffold" "github.com/seaweedfs/seaweedfs/weed/command/scaffold"
) )
func init() { func init() {

42
weed/command/server.go
View file

@ -7,12 +7,12 @@ import (
"strings" "strings"
"time" "time"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/seaweedfs/seaweedfs/weed/util/grace"
) )
type ServerOptions struct { type ServerOptions struct {
@ -24,13 +24,13 @@ type ServerOptions struct {
} }
var ( var (
serverOptions ServerOptions serverOptions ServerOptions
masterOptions MasterOptions masterOptions MasterOptions
filerOptions FilerOptions filerOptions FilerOptions
s3Options S3Options s3Options S3Options
iamOptions IamOptions iamOptions IamOptions
webdavOptions WebDavOption webdavOptions WebDavOption
msgBrokerOptions MessageBrokerOptions mqBrokerOptions MessageQueueBrokerOptions
) )
func init() { func init() {
@ -74,7 +74,7 @@ var (
isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service") isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway") isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") isStartingMqBroker = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
serverWhiteList []string serverWhiteList []string
@ -155,7 +155,7 @@ func init() {
webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
} }
@ -179,7 +179,7 @@ func runServer(cmd *Command, args []string) bool {
if *isStartingWebDav { if *isStartingWebDav {
*isStartingFiler = true *isStartingFiler = true
} }
if *isStartingMsgBroker { if *isStartingMqBroker {
*isStartingFiler = true *isStartingFiler = true
} }
@ -208,7 +208,9 @@ func runServer(cmd *Command, args []string) bool {
serverOptions.v.idleConnectionTimeout = serverTimeout serverOptions.v.idleConnectionTimeout = serverTimeout
serverOptions.v.dataCenter = serverDataCenter serverOptions.v.dataCenter = serverDataCenter
serverOptions.v.rack = serverRack serverOptions.v.rack = serverRack
msgBrokerOptions.ip = serverIp mqBrokerOptions.ip = serverIp
mqBrokerOptions.masters = filerOptions.masters
mqBrokerOptions.filerGroup = filerOptions.filerGroup
// serverOptions.v.pulseSeconds = pulseSeconds // serverOptions.v.pulseSeconds = pulseSeconds
// masterOptions.pulseSeconds = pulseSeconds // masterOptions.pulseSeconds = pulseSeconds
@ -217,6 +219,8 @@ func runServer(cmd *Command, args []string) bool {
filerOptions.dataCenter = serverDataCenter filerOptions.dataCenter = serverDataCenter
filerOptions.rack = serverRack filerOptions.rack = serverRack
mqBrokerOptions.dataCenter = serverDataCenter
mqBrokerOptions.rack = serverRack
filerOptions.disableHttp = serverDisableHttp filerOptions.disableHttp = serverDisableHttp
masterOptions.disableHttp = serverDisableHttp masterOptions.disableHttp = serverDisableHttp
@ -224,7 +228,7 @@ func runServer(cmd *Command, args []string) bool {
s3Options.filer = &filerAddress s3Options.filer = &filerAddress
iamOptions.filer = &filerAddress iamOptions.filer = &filerAddress
webdavOptions.filer = &filerAddress webdavOptions.filer = &filerAddress
msgBrokerOptions.filer = &filerAddress mqBrokerOptions.filerGroup = filerOptions.filerGroup
go stats_collect.StartMetricsServer(*serverMetricsHttpPort) go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
@ -276,10 +280,10 @@ func runServer(cmd *Command, args []string) bool {
}() }()
} }
if *isStartingMsgBroker { if *isStartingMqBroker {
go func() { go func() {
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
msgBrokerOptions.startQueueServer() mqBrokerOptions.startQueueServer()
}() }()
} }

8
weed/command/shell.go
View file

@ -2,11 +2,11 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/shell" "github.com/seaweedfs/seaweedfs/weed/shell"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

10
weed/command/update.go
View file

@ -19,8 +19,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"golang.org/x/net/context/ctxhttp" "golang.org/x/net/context/ctxhttp"
) )
@ -76,8 +76,8 @@ func init() {
var cmdUpdate = &Command{ var cmdUpdate = &Command{
UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]", UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]",
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs", Short: "get latest or specific version from https://github.com/seaweedfs/seaweedfs",
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`, Long: `get latest or specific version from https://github.com/seaweedfs/seaweedfs`,
} }
func runUpdate(cmd *Command, args []string) bool { func runUpdate(cmd *Command, args []string) bool {
@ -118,7 +118,7 @@ func runUpdate(cmd *Command, args []string) bool {
func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) { func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
currentVersion := util.VERSION_NUMBER currentVersion := util.VERSION_NUMBER
rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs") rel, err := GitHubLatestRelease(ctx, ver, "seaweedfs", "seaweedfs")
if err != nil { if err != nil {
return "", err return "", err
} }

10
weed/command/upload.go
View file

@ -4,15 +4,15 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc" "google.golang.org/grpc"
"os" "os"
"path/filepath" "path/filepath"
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

2
weed/command/version.go
View file

@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"runtime" "runtime"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var cmdVersion = &Command{ var cmdVersion = &Command{

22
weed/command/volume.go
View file

@ -10,25 +10,25 @@ import (
"strings" "strings"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/spf13/viper" "github.com/spf13/viper"
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/util/grace" "github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util/httpdown" "github.com/seaweedfs/seaweedfs/weed/util/httpdown"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
weed_server "github.com/chrislusf/seaweedfs/weed/server" weed_server "github.com/seaweedfs/seaweedfs/weed/server"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

2
weed/command/volume_test.go
View file

@ -5,7 +5,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
) )
func TestXYZ(t *testing.T) { func TestXYZ(t *testing.T) {

12
weed/command/webdav.go
View file

@ -9,12 +9,12 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server" "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

8
weed/filer/abstract_sql/abstract_sql_store.go
View file

@ -4,10 +4,10 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"strings" "strings"
"sync" "sync"
) )

6
weed/filer/abstract_sql/abstract_sql_store_kv.go
View file

@ -5,9 +5,9 @@ import (
"database/sql" "database/sql"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"strings" "strings"
) )

8
weed/filer/arangodb/arangodb_store.go
View file

@ -11,10 +11,10 @@ import (
"github.com/arangodb/go-driver" "github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/http" "github.com/arangodb/go-driver/http"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

4
weed/filer/arangodb/arangodb_store_bucket.go
View file

@ -5,9 +5,9 @@ import (
"github.com/arangodb/go-driver" "github.com/arangodb/go-driver"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
) )
var _ filer.BucketAware = (*ArangodbStore)(nil) var _ filer.BucketAware = (*ArangodbStore)(nil)

4
weed/filer/arangodb/arangodb_store_kv.go
View file

@ -5,8 +5,8 @@ import (
"fmt" "fmt"
"github.com/arangodb/go-driver" "github.com/arangodb/go-driver"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
) )
func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

2
weed/filer/arangodb/helpers.go
View file

@ -9,7 +9,7 @@ import (
"strings" "strings"
"github.com/arangodb/go-driver" "github.com/arangodb/go-driver"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
//convert a string into arango-key safe hex bytes hash //convert a string into arango-key safe hex bytes hash

8
weed/filer/cassandra/cassandra_store.go
View file

@ -6,10 +6,10 @@ import (
"github.com/gocql/gocql" "github.com/gocql/gocql"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func init() { func init() {

2
weed/filer/cassandra/cassandra_store_kv.go
View file

@ -4,8 +4,8 @@ import (
"context" "context"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/gocql/gocql" "github.com/gocql/gocql"
"github.com/seaweedfs/seaweedfs/weed/filer"
) )
func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

4
weed/filer/configuration.go
View file

@ -1,8 +1,8 @@
package filer package filer
import ( import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"os" "os"
"reflect" "reflect"
"strings" "strings"

8
weed/filer/elastic/v7/elastic_store.go
View file

@ -9,12 +9,12 @@ import (
"math" "math"
"strings" "strings"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7" elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
) )
var ( var (

4
weed/filer/elastic/v7/elastic_store_kv.go
View file

@ -7,11 +7,11 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7" elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/glog"
) )
func (store *ElasticStore) KvDelete(ctx context.Context, key []byte) (err error) { func (store *ElasticStore) KvDelete(ctx context.Context, key []byte) (err error) {

4
weed/filer/entry.go
View file

@ -4,8 +4,8 @@ import (
"os" "os"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
type Attr struct { type Attr struct {

2
weed/filer/entry_codec.go
View file

@ -8,7 +8,7 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
) )
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {

8
weed/filer/etcd/etcd_store.go
View file

@ -9,10 +9,10 @@ import (
"go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util" weed_util "github.com/seaweedfs/seaweedfs/weed/util"
) )
const ( const (

2
weed/filer/etcd/etcd_store_kv.go
View file

@ -3,7 +3,7 @@ package etcd
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer"
) )
func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

2
weed/filer/etcd/etcd_store_test.go
View file

@ -1,7 +1,7 @@
package etcd package etcd
import ( import (
"github.com/chrislusf/seaweedfs/weed/filer/store_test" "github.com/seaweedfs/seaweedfs/weed/filer/store_test"
"testing" "testing"
) )

8
weed/filer/filechunk_manifest.go
View file

@ -10,13 +10,13 @@ import (
"sync" "sync"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
const ( const (

2
weed/filer/filechunk_manifest_test.go
View file

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
) )
func TestDoMaybeManifestize(t *testing.T) { func TestDoMaybeManifestize(t *testing.T) {

11
weed/filer/filechunks.go
View file

@ -3,12 +3,12 @@ package filer
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"math" "math"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) { func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
@ -114,9 +114,12 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
fileIds := make(map[string]bool) fileIds := make(map[string]bool)
for _, interval := range bs { for _, interval := range bs {
fileIds[interval.GetFileIdString()] = true fileIds[interval.GetFileIdString()] = true
fileIds[interval.GetSourceFileId()] = true
} }
for _, chunk := range as { for _, chunk := range as {
if _, found := fileIds[chunk.GetSourceFileId()]; !found { _, sourceFileIdFound := fileIds[chunk.GetSourceFileId()]
_, fileIdFound := fileIds[chunk.GetFileId()]
if !sourceFileIdFound && !fileIdFound {
delta = append(delta, chunk) delta = append(delta, chunk)
} }
} }

43
weed/filer/filechunks2_test.go
View file

@ -1,13 +1,52 @@
package filer package filer
import ( import (
"github.com/stretchr/testify/assert"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"log"
"testing" "testing"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
) )
func TestDoMinusChunks(t *testing.T) {
// https://github.com/seaweedfs/seaweedfs/issues/3328
// clusterA and clusterB using filer.sync to sync file: hello.txt
// clusterA append a new line and then clusterB also append a new line
// clusterA append a new line again
chunksInA := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "33", Mtime: 300},
}
chunksInB := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", Mtime: 300},
}
// clusterB using command "echo 'content' > hello.txt" to overwrite file
// clusterA will receive two evenNotification, need to empty the whole file content first and add new content
// the first one is oldEntry is chunksInB and newEntry is empty fileChunks
firstOldEntry := chunksInB
var firstNewEntry []*filer_pb.FileChunk
// clusterA received the first one event, gonna empty the whole chunk, according the code in filer_sink 194
// we can get the deleted chunks and newChunks
firstDeletedChunks := DoMinusChunks(firstOldEntry, firstNewEntry)
log.Println("first deleted chunks:", firstDeletedChunks)
//firstNewEntry := DoMinusChunks(firstNewEntry, firstOldEntry)
// clusterA need to delete all chunks in firstDeletedChunks
emptiedChunksInA := DoMinusChunksBySourceFileId(chunksInA, firstDeletedChunks)
// chunksInA supposed to be empty by minus the deletedChunks but it just delete the chunk which sync from clusterB
log.Println("clusterA synced empty chunks event result:", emptiedChunksInA)
// clusterB emptied it's chunks and clusterA must sync the change and empty chunks too
assert.Equalf(t, firstNewEntry, emptiedChunksInA, "empty")
}
func TestCompactFileChunksRealCase(t *testing.T) { func TestCompactFileChunksRealCase(t *testing.T) {
chunks := []*filer_pb.FileChunk{ chunks := []*filer_pb.FileChunk{

2
weed/filer/filechunks_read.go
View file

@ -1,7 +1,7 @@
package filer package filer
import ( import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
) )

2
weed/filer/filechunks_read_test.go
View file

@ -2,7 +2,7 @@ package filer
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"math/rand" "math/rand"
"testing" "testing"
) )

2
weed/filer/filechunks_test.go
View file

@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
) )
func TestCompactFileChunks(t *testing.T) { func TestCompactFileChunks(t *testing.T) {

41
weed/filer/filer.go
View file

@ -3,9 +3,9 @@ package filer
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"os" "os"
"sort" "sort"
"strings" "strings"
@ -13,11 +13,11 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/log_buffer" "github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
) )
const ( const (
@ -52,7 +52,7 @@ type Filer struct {
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer { filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
f := &Filer{ f := &Filer{
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, masters), MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
fileIdDeletionQueue: util.NewUnboundedQueue(), fileIdDeletionQueue: util.NewUnboundedQueue(),
GrpcDialOption: grpcDialOption, GrpcDialOption: grpcDialOption,
FilerConf: NewFilerConf(), FilerConf: NewFilerConf(),
@ -105,28 +105,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
} }
func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) { func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: cluster.FilerType,
FilerGroup: f.MasterClient.FilerGroup,
})
glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
for _, node := range resp.ClusterNodes {
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
NodeType: cluster.FilerType,
Address: node.Address,
IsLeader: node.IsLeader,
IsAdd: true,
CreatedAtNs: node.CreatedAtNs,
})
}
return err
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr)
}
return
} }
func (f *Filer) SetStore(store FilerStore) (isFresh bool) { func (f *Filer) SetStore(store FilerStore) (isFresh bool) {

10
weed/filer/filer_conf.go
View file

@ -4,15 +4,15 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc" "google.golang.org/grpc"
"io" "io"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/jsonpb"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/viant/ptrie" "github.com/viant/ptrie"
) )

2
weed/filer/filer_conf_test.go
View file

@ -3,7 +3,7 @@ package filer
import ( import (
"testing" "testing"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )

Some files were not shown because too many files have changed in this diff Show more