Merge pull request #6 from chrislusf/master

This commit is contained in:
bingoohuang 2021-04-26 17:15:17 +08:00 committed by GitHub
commit 4ee58922ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
231 changed files with 10136 additions and 2144 deletions

View file

@ -40,6 +40,7 @@ jobs:
goarch: ${{ matrix.goarch }} goarch: ${{ matrix.goarch }}
release_tag: dev release_tag: dev
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0
build_flags: -tags 5BytesOffset # optional, default is build_flags: -tags 5BytesOffset # optional, default is
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
@ -55,8 +56,9 @@ jobs:
goarch: ${{ matrix.goarch }} goarch: ${{ matrix.goarch }}
release_tag: dev release_tag: dev
overwrite: true overwrite: true
pre_command: export CGO_ENABLED=0
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
# Where to run `go build .` # Where to run `go build .`
project_path: weed project_path: weed
binary_name: weed- binary_name: weed
asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

View file

@ -7,6 +7,7 @@
[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed) [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki) [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/) [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
[![SeaweedFS on Maven Central](https://img.shields.io/maven-central/v/com.github.chrislusf/seaweedfs-client)](https://search.maven.org/search?q=g:com.github.chrislusf)
![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png) ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
@ -42,6 +43,7 @@ Your support will be really appreciated by me and other supporters!
- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS) - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs) - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki) - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
- [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction) - [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
Table of Contents Table of Contents
@ -70,7 +72,7 @@ Table of Contents
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thoudsands of machines. That is it! Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
## Introduction ## ## Introduction ##
@ -79,17 +81,34 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
1. to store billions of files! 1. to store billions of files!
2. to serve the files fast! 2. to serve the files fast!
SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages volumes on volume servers, and these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation). SeaweedFS started as an Object Store to handle small files efficiently.
Instead of managing all file metadata in a central master,
the central master only manages volumes on volume servers,
and these volume servers manage files and their metadata.
This relieves concurrency pressure from the central master and spreads file metadata into volume servers,
allowing faster file access (O(1), usually just one disk read operation).
SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity. What's more, the cloud storage access API cost is minimized. Faster and Cheaper than direct cloud storage! SeaweedFS can transparently integrate with the cloud.
With hot data on local cluster, and warm data on the cloud with O(1) access time,
SeaweedFS can achieve both fast local access time and elastic cloud storage capacity.
What's more, the cloud storage access API cost is minimized.
Faster and Cheaper than direct cloud storage!
Signup for future managed SeaweedFS cluster offering at "seaweedfilesystem at gmail dot com".
There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases. There is only 40 bytes of disk storage overhead for each file's metadata.
It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebooks Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf) SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
Also, SeaweedFS implements erasure coding with ideas from
[f4: Facebooks Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebooks Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc. On top of the object store, optional [Filer] can support directories and POSIX attributes.
Filer is a separate linearly-scalable stateless server with customizable metadata stores,
e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, MemSql, TiDB, Etcd, CockroachDB, etc.
For any distributed key value stores, the large values can be offloaded to SeaweedFS. With the fast access speed and linearly scalable capacity, SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore]. For any distributed key value stores, the large values can be offloaded to SeaweedFS.
With the fast access speed and linearly scalable capacity,
SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
[Back to TOC](#table-of-contents) [Back to TOC](#table-of-contents)
@ -105,6 +124,7 @@ For any distributed key value stores, the large values can be offloaded to Seawe
* Support ETag, Accept-Range, Last-Modified, etc. * Support ETag, Accept-Range, Last-Modified, etc.
* Support in-memory/leveldb/readonly mode tuning for memory/performance balance. * Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
* Support rebalancing the writable and readonly volumes. * Support rebalancing the writable and readonly volumes.
* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data. * [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability. * [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
@ -135,6 +155,7 @@ For any distributed key value stores, the large values can be offloaded to Seawe
[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV [WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage [ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
[TieredStorage]: https://github.com/chrislusf/seaweedfs/wiki/Tiered-Storage
[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier [CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption [FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores [FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores

View file

@ -1,3 +1,4 @@
<h1 align="center">Sponsors &amp; Backers</h1> <h1 align="center">Sponsors &amp; Backers</h1>
- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs). - [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
@ -12,3 +13,4 @@
- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/) - [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
- [Haravan - Ecommerce Platform](https://www.haravan.com) - [Haravan - Ecommerce Platform](https://www.haravan.com)
- PeterCxy - Creator of Shelter App

View file

@ -1,6 +1,7 @@
FROM alpine FROM alpine
ARG RELEASE=latest # 'latest' or 'dev' # 'latest' or 'dev'
ARG RELEASE=latest
RUN \ RUN \
ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \ ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \

View file

@ -6,7 +6,7 @@ ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -ldflags "${LDFLAGS}" && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}"
FROM alpine AS final FROM alpine AS final
LABEL author="Chris Lu" LABEL author="Chris Lu"

View file

@ -6,7 +6,7 @@ ARG BRANCH=${BRANCH:-master}
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \ RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
&& go install -tags 5BytesOffset -ldflags "${LDFLAGS}" && CGO_ENABLED=0 go install -tags 5BytesOffset -ldflags "-extldflags -static ${LDFLAGS}"
FROM alpine AS final FROM alpine AS final
LABEL author="Chris Lu" LABEL author="Chris Lu"

View file

@ -28,4 +28,4 @@ ENV \
S3TEST_CONF="/s3test.conf" S3TEST_CONF="/s3test.conf"
ENTRYPOINT ["/bin/bash", "-c"] ENTRYPOINT ["/bin/bash", "-c"]
CMD ["exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] CMD ["sleep 10 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"]

View file

@ -4,8 +4,10 @@ all: gen
gen: dev gen: dev
build: binary:
cd ../weed; GOOS=linux go build; mv weed ../docker/ cd ../weed; CGO_ENABLED=0 GOOS=linux go build -ldflags "-extldflags -static"; mv weed ../docker/
build: binary
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local . docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
rm ./weed rm ./weed
@ -15,9 +17,15 @@ s3tests_build:
dev: build dev: build
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_tls: build certstrap
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_mount: build dev_mount: build
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
profile_mount: build
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
k8s: build k8s: build
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
@ -41,3 +49,15 @@ filer_etcd: build
clean: clean:
rm ./weed rm ./weed
certstrap:
go get github.com/square/certstrap
certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true

0
docker/compose/dev.env Normal file
View file

View file

@ -11,6 +11,10 @@ services:
- 8888:8888 - 8888:8888
- 18888:18888 - 18888:18888
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./master-cloud.toml:/etc/seaweedfs/master.toml
depends_on:
- server2
server2: server2:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
ports: ports:

View file

@ -6,33 +6,50 @@ services:
ports: ports:
- 9333:9333 - 9333:9333
- 19333:19333 - 19333:19333
command: "master -ip=master" command: "-v=1 master -ip=master"
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
volume: volume:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
ports: ports:
- 8080:8080 - 8080:8080
- 18080:18080 - 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1" command: "-v=1 volume -mserver=master:9333 -port=8080 -ip=volume -preStopSeconds=1"
depends_on: depends_on:
- master - master
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
filer: filer:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
ports: ports:
- 8111:8111
- 8888:8888 - 8888:8888
- 18888:18888 - 18888:18888
command: 'filer -master="master:9333"' command: '-v=1 filer -master="master:9333" -iam'
depends_on: depends_on:
- master - master
- volume - volume
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
s3: s3:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
ports: ports:
- 8333:8333 - 8333:8333
command: 's3 -filer="filer:8888"' command: '-v=1 s3 -filer="filer:8888"'
depends_on: depends_on:
- master - master
- volume - volume
- filer - filer
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
mount: mount:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
privileged: true privileged: true
@ -40,6 +57,10 @@ services:
- SYS_ADMIN - SYS_ADMIN
mem_limit: 4096m mem_limit: 4096m
command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128' command: '-v=4 mount -filer="filer:8888" -dirAutoCreate -dir=/mnt/seaweedfs -cacheCapacityMB=100 -concurrentWriters=128'
volumes:
- ./tls:/etc/seaweedfs/tls
env_file:
- ${ENV_FILE:-dev.env}
depends_on: depends_on:
- master - master
- volume - volume

View file

@ -0,0 +1,50 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
- 8080:8080
- 18080:18080
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
depends_on:
- master
s3:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 8333:8333
command: '-v 1 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on:
- master
- volume
minio-gateway-s3:
image: minio/minio
ports:
- 9000:9000
command: 'minio gateway s3 http://s3:8333'
restart: on-failure
environment:
MINIO_ACCESS_KEY: "some_access_key1"
MINIO_SECRET_KEY: "some_secret_key1"
depends_on:
- s3
minio-warp:
image: minio/warp
command: 'mixed --duration=5m --obj.size=3mb --autoterm'
restart: on-failure
environment:
WARP_HOST: "minio-gateway-s3:9000"
WARP_ACCESS_KEY: "some_access_key1"
WARP_SECRET_KEY: "some_secret_key1"
depends_on:
- minio-gateway-s3

View file

@ -0,0 +1,47 @@
version: '2'
services:
master:
image: chrislusf/seaweedfs:local
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master"
volume:
image: chrislusf/seaweedfs:local
ports:
- 7455:8080
- 9325:9325
volumes:
- /Volumes/mobile_disk/99:/data
command: 'volume -mserver="master:9333" -port=8080 -metricsPort=9325 -preStopSeconds=1 -publicUrl=localhost:7455'
depends_on:
- master
filer:
image: chrislusf/seaweedfs:local
ports:
- 8888:8888
- 18888:18888
- 9326:9326
volumes:
- /Volumes/mobile_disk/99:/data
command: 'filer -master="master:9333" -metricsPort=9326'
tty: true
stdin_open: true
depends_on:
- master
- volume
mount:
image: chrislusf/seaweedfs:local
privileged: true
cap_add:
- SYS_ADMIN
devices:
- fuse
volumes:
- /Volumes/mobile_disk/99:/data
entrypoint: '/bin/sh -c "mkdir -p t1 && weed mount -filer=filer:8888 -dir=./t1 -cacheCapacityMB=0 -memprofile=/data/mount.mem.pprof"'
depends_on:
- master
- volume
- filer

View file

@ -15,24 +15,18 @@ services:
command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1" command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
depends_on: depends_on:
- master - master
filer: s3:
image: chrislusf/seaweedfs:local image: chrislusf/seaweedfs:local
ports: ports:
- 8888:8888 - 8888:8888
- 18888:18888 - 18888:18888
command: 'filer -master="master:9333"'
depends_on:
- master
- volume
s3:
image: chrislusf/seaweedfs:local
ports:
- 8333:8333 - 8333:8333
command: '-v 9 s3 -filer="filer:8888"' command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
depends_on: depends_on:
- master - master
- volume - volume
- filer
minio: minio:
image: minio/minio image: minio/minio
ports: ports:

View file

@ -38,7 +38,7 @@ services:
S3TEST_CONF: "s3tests.conf" S3TEST_CONF: "s3tests.conf"
NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3" NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy" NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_delimiter_not_skip_special|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_head_zero_bytes|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_authenticated|object_raw_response_headers|object_raw_authenticated_bucket_acl|object_raw_authenticated_object_acl|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_not_expired|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_create_exists|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|bucket_create_special_key_names|object_copy_zero_size|object_copy_verify_contenttype|object_copy_to_itself|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)" NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_write_cache_control|object_write_expires|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifmatch_failed|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
depends_on: depends_on:
- master - master
- volume - volume

View file

@ -0,0 +1,30 @@
# Put this file to one of the location, with descending priority
# ./master.toml
# $HOME/.seaweedfs/master.toml
# /etc/seaweedfs/master.toml
# this file is read by master
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -force
ec.balance -force
volume.balance -force
volume.fix.replication
unlock
"""
sleep_minutes = 17 # sleep minutes between each script execution
# configurations for tiered cloud storage
# old volumes are transparently moved to cloud for cost efficiency
[storage.backend]
[storage.backend.s3.default]
enabled = true
aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
region = "us-east-2"
bucket = "volume_bucket" # an existing bucket
endpoint = "http://server2:8333"

14
docker/compose/tls.env Normal file
View file

@ -0,0 +1,14 @@
WEED_GRPC_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
WEED_GRPC_ALLOWED_WILDCARD_DOMAIN=".dev"
WEED_GRPC_MASTER_CERT=/etc/seaweedfs/tls/master01.dev.crt
WEED_GRPC_MASTER_KEY=/etc/seaweedfs/tls/master01.dev.key
WEED_GRPC_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
WEED_GRPC_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
WEED_GRPC_FILER_CERT=/etc/seaweedfs/tls/filer01.dev.crt
WEED_GRPC_FILER_KEY=/etc/seaweedfs/tls/filer01.dev.key
WEED_GRPC_CLIENT_CERT=/etc/seaweedfs/tls/client01.dev.crt
WEED_GRPC_CLIENT_KEY=/etc/seaweedfs/tls/client01.dev.key
WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"

View file

@ -60,9 +60,9 @@ case "$1" in
'cronjob') 'cronjob')
MASTER=${WEED_MASTER-localhost:9333} MASTER=${WEED_MASTER-localhost:9333}
FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *} FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab
BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *} BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab
echo "Running Crontab:" echo "Running Crontab:"
cat /crontab cat /crontab
exec supercronic /crontab exec supercronic /crontab

11
go.mod
View file

@ -13,8 +13,9 @@ require (
github.com/Shopify/sarama v1.23.1 github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.34.30 github.com/aws/aws-sdk-go v1.34.30
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash v1.1.0
github.com/chrislusf/raft v1.0.4 github.com/chrislusf/raft v1.0.6
github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.2 github.com/disintegration/imaging v1.6.2
@ -38,8 +39,10 @@ require (
github.com/google/uuid v1.1.1 github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.4 github.com/gorilla/mux v1.7.4
github.com/gorilla/websocket v1.4.1 // indirect github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/copier v0.2.8
github.com/json-iterator/go v1.1.10 github.com/json-iterator/go v1.1.10
github.com/karlseguin/ccache v2.0.3+incompatible // indirect github.com/karlseguin/ccache v2.0.3+incompatible // indirect
github.com/karlseguin/ccache/v2 v2.0.7 github.com/karlseguin/ccache/v2 v2.0.7
@ -48,7 +51,7 @@ require (
github.com/klauspost/crc32 v1.2.0 github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2 github.com/klauspost/reedsolomon v1.9.2
github.com/kurin/blazer v0.5.3 github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.2.0 github.com/lib/pq v1.10.0
github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/lunixbochs/vtclean v1.0.0 // indirect
github.com/magiconair/properties v1.8.1 // indirect github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-colorable v0.1.2 // indirect
@ -59,7 +62,7 @@ require (
github.com/prometheus/client_golang v1.3.0 github.com/prometheus/client_golang v1.3.0
github.com/rakyll/statik v0.1.7 github.com/rakyll/statik v0.1.7
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
github.com/seaweedfs/fuse v1.1.1 github.com/seaweedfs/fuse v1.1.4
github.com/seaweedfs/goexif v1.0.2 github.com/seaweedfs/goexif v1.0.2
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
@ -87,7 +90,7 @@ require (
gocloud.dev/pubsub/rabbitpubsub v0.20.0 gocloud.dev/pubsub/rabbitpubsub v0.20.0
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd
golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 golang.org/x/tools v0.0.0-20200608174601-1b747fd94509
google.golang.org/api v0.26.0 google.golang.org/api v0.26.0

16
go.sum
View file

@ -141,6 +141,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA=
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg= github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
@ -153,6 +155,10 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM= github.com/chrislusf/raft v1.0.4 h1:THhbsVik2hxdE0/VXX834f64Wn9RzgVPp+E+XCWZdKM=
github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.4/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
github.com/chrislusf/raft v1.0.5 h1:g8GxKCSStfm0/bGBDpNEbmEXL6MJkpXX+NI0ksbX5D4=
github.com/chrislusf/raft v1.0.5/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
github.com/chrislusf/raft v1.0.6 h1:wunb85WWhMKhNRn7EmdIw35D4Lmew0ZJv8oYDizR/+Y=
github.com/chrislusf/raft v1.0.6/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@ -433,6 +439,8 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjL
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@ -496,6 +504,8 @@ github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8= github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8=
@ -681,6 +691,10 @@ github.com/seaweedfs/fuse v1.1.0 h1:cL1qPHFNtFv0UuJTLjKKgWDzfJ4iZzTa4Y7ipc2acGw=
github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/fuse v1.1.0/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0= github.com/seaweedfs/fuse v1.1.1 h1:WD51YFJcBViOx8I89jeqPD+vAKl4EowzBy9GUw0plb0=
github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8= github.com/seaweedfs/fuse v1.1.1/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
github.com/seaweedfs/fuse v1.1.3 h1:0DddotXwSRGbYG2kynoJyr8GHCy30Z2SpdhP3vdyijY=
github.com/seaweedfs/fuse v1.1.3/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
github.com/seaweedfs/fuse v1.1.4 h1:YYqkK86agMhXRSwR+wFbRI8ikMgk3kL6PNTna1MAHyQ=
github.com/seaweedfs/fuse v1.1.4/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E= github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk= github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw= github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
@ -955,6 +969,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0= golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6cnrWpidecD1UuSYbhKm0=
golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

View file

@ -1,5 +1,5 @@
apiVersion: v1 apiVersion: v1
description: SeaweedFS description: SeaweedFS
name: seaweedfs name: seaweedfs
appVersion: "2.28" appVersion: "2.41"
version: 2.28 version: 2.41

View file

@ -23,6 +23,6 @@ spec:
targetPort: {{ .Values.volume.metricsPort }} targetPort: {{ .Values.volume.metricsPort }}
protocol: TCP protocol: TCP
{{- end }} {{- end }}
selector: selector:
app: {{ template "seaweedfs.name" . }} app: {{ template "seaweedfs.name" . }}
component: volume component: volume

View file

@ -4,7 +4,7 @@ global:
registry: "" registry: ""
repository: "" repository: ""
imageName: chrislusf/seaweedfs imageName: chrislusf/seaweedfs
# imageTag: "2.28" - started using {.Chart.appVersion} # imageTag: "2.41" - started using {.Chart.appVersion}
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret imagePullSecrets: imagepullsecret
restartPolicy: Always restartPolicy: Always

Binary file not shown.

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 86 KiB

BIN
note/SeaweedFS_XDR.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.2</version> <version>1.6.4</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>
@ -17,7 +17,7 @@
<protobuf.version>3.9.1</protobuf.version> <protobuf.version>3.9.1</protobuf.version>
<!-- follow https://github.com/grpc/grpc-java --> <!-- follow https://github.com/grpc/grpc-java -->
<grpc.version>1.23.0</grpc.version> <grpc.version>1.23.0</grpc.version>
<guava.version>28.0-jre</guava.version> <guava.version>30.0-jre</guava.version>
</properties> </properties>
<dependencies> <dependencies>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.2</version> <version>1.6.4</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.2</version> <version>1.6.4</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -3,6 +3,7 @@ package seaweedfs.client;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.nio.Buffer;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -34,7 +35,7 @@ public class ByteBufferPool {
} }
public static synchronized void release(ByteBuffer obj) { public static synchronized void release(ByteBuffer obj) {
obj.clear(); ((Buffer)obj).clear();
bufferList.add(0, obj); bufferList.add(0, obj);
} }

View file

@ -94,7 +94,7 @@ public class FilerClient extends FilerGrpcClient {
return true; return true;
} }
File pathFile = new File(path); File pathFile = new File(path);
String parent = pathFile.getParent(); String parent = pathFile.getParent().replace('\\','/');
String name = pathFile.getName(); String name = pathFile.getName();
mkdirs(parent, mode, uid, gid, userName, groupNames); mkdirs(parent, mode, uid, gid, userName, groupNames);
@ -115,11 +115,11 @@ public class FilerClient extends FilerGrpcClient {
public boolean mv(String oldPath, String newPath) { public boolean mv(String oldPath, String newPath) {
File oldPathFile = new File(oldPath); File oldPathFile = new File(oldPath);
String oldParent = oldPathFile.getParent(); String oldParent = oldPathFile.getParent().replace('\\','/');
String oldName = oldPathFile.getName(); String oldName = oldPathFile.getName();
File newPathFile = new File(newPath); File newPathFile = new File(newPath);
String newParent = newPathFile.getParent(); String newParent = newPathFile.getParent().replace('\\','/');
String newName = newPathFile.getName(); String newName = newPathFile.getName();
return atomicRenameEntry(oldParent, oldName, newParent, newName); return atomicRenameEntry(oldParent, oldName, newParent, newName);
@ -129,7 +129,7 @@ public class FilerClient extends FilerGrpcClient {
public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) { public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) {
File pathFile = new File(path); File pathFile = new File(path);
String parent = pathFile.getParent(); String parent = pathFile.getParent().replace('\\','/');
String name = pathFile.getName(); String name = pathFile.getName();
return deleteEntry( return deleteEntry(
@ -148,7 +148,7 @@ public class FilerClient extends FilerGrpcClient {
public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) { public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
File pathFile = new File(path); File pathFile = new File(path);
String parent = pathFile.getParent(); String parent = pathFile.getParent().replace('\\','/');
String name = pathFile.getName(); String name = pathFile.getName();
FilerProto.Entry entry = lookupEntry(parent, name); FilerProto.Entry entry = lookupEntry(parent, name);

View file

@ -11,13 +11,13 @@
<dependency> <dependency>
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.6.2</version> <version>1.6.4</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId> <artifactId>seaweedfs-hadoop2-client</artifactId>
<version>1.6.2</version> <version>1.6.4</version>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>

View file

@ -0,0 +1,120 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>hdfs-over-ftp</groupId>
<artifactId>hdfs-over-ftp</artifactId>
<version>1.0</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.4.3</version>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.ftpserver</groupId>
<artifactId>ftpserver-core</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop3-client</artifactId>
<version>1.6.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<compilerArguments>
<verbose />
<bootclasspath>${java.home}/lib/rt.jar</bootclasspath>
</compilerArguments>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.6</version>
<configuration>
<archive>
<manifest>
<mainClass>org.apache.hadoop.seaweed.ftp.ApplicationServer</mainClass>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
</manifest>
<manifestEntries>
<Class-Path>./</Class-Path>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/resources/assembly.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View file

@ -0,0 +1,14 @@
package org.apache.hadoop.seaweed.ftp;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class ApplicationServer {
public static void main(String[] args) {
SpringApplication.run(ApplicationServer.class, args);
}
}

View file

@ -0,0 +1,27 @@
package org.apache.hadoop.seaweed.ftp.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
public class SwaggerConfig {
@Bean
public Docket createRestApi() {
return new Docket(DocumentationType.SWAGGER_2)
.pathMapping("/")
.select()
.apis(RequestHandlerSelectors.basePackage("org.apache.hadoop.seaweed.ftp"))
.paths(PathSelectors.any())
.build().apiInfo(new ApiInfoBuilder()
.title("FTP API Doc")
.version("1.0")
.build());
}
}

View file

@ -0,0 +1,71 @@
package org.apache.hadoop.seaweed.ftp.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.hadoop.seaweed.ftp.service.HFtpService;
import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
@RestController
@RequestMapping("/manager")
@Api(tags = "FTP操作管理")
public class FtpManagerController {
private static Logger log = Logger.getLogger(FtpManagerController.class);
@Autowired
private HFtpService hdfsOverFtpServer;
@GetMapping("/status")
@ApiOperation("查看FTP服务状态")
public Result status() {
Map map = new HashMap<>();
try {
boolean status = hdfsOverFtpServer.statusServer();
map.put("is_running", status);
return new Result(true, map, "FTP 服务状态获取成功");
}catch (Exception e) {
log.error(e);
map.put("is_running", false);
return new Result(true, map, "FTP 服务状态获取成功");
}
}
@PutMapping("/start")
@ApiOperation("启动FTP服务")
public Result start() {
try {
boolean status = hdfsOverFtpServer.statusServer();
if(!status) {
hdfsOverFtpServer.startServer();
}
return new Result(true, "FTP 服务启动成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "FTP 服务启动失败");
}
}
@PutMapping("/stop")
@ApiOperation("停止FTP服务")
public Result stop() {
try {
boolean status = hdfsOverFtpServer.statusServer();
if(status) {
hdfsOverFtpServer.stopServer();
}
return new Result(true, "FTP 服务停止成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "FTP 服务停止失败");
}
}
}

View file

@ -0,0 +1,98 @@
package org.apache.hadoop.seaweed.ftp.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.ftpserver.ftplet.User;
import org.apache.ftpserver.usermanager.Md5PasswordEncryptor;
import org.apache.ftpserver.usermanager.UserFactory;
import org.apache.hadoop.seaweed.ftp.controller.vo.FtpUser;
import org.apache.hadoop.seaweed.ftp.controller.vo.Result;
import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
import org.apache.log4j.Logger;
import org.springframework.web.bind.annotation.*;
import java.io.File;
@RestController
@RequestMapping("/user")
@Api(tags = "FTP用户管理")
public class UserController {
private static Logger log = Logger.getLogger(UserController.class);
/***
* {
* "name": "test",
* "password": "test",
* "homeDirectory": "/buckets/test/"
* }
* @param ftpUser
* @return
*/
@PostMapping("/add")
@ApiOperation("新增/编辑用户")
public Result add(@RequestBody FtpUser ftpUser) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
userManagerFactory.setPasswordEncryptor(new Md5PasswordEncryptor());
UserFactory userFactory = new UserFactory();
userFactory.setHomeDirectory(ftpUser.getHomeDirectory());
userFactory.setName(ftpUser.getName());
userFactory.setPassword(ftpUser.getPassword());
userFactory.setEnabled(ftpUser.isEnabled());
userFactory.setMaxIdleTime(ftpUser.getMaxIdleTime());
User user = userFactory.createUser();
userManagerFactory.save(user, ftpUser.isRenamePush());
return new Result(true, "新建用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "新建用户失败");
}
}
@DeleteMapping("/delete/{user}")
@ApiOperation("删除用户")
public Result delete(@PathVariable(value = "user") String user) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
userManagerFactory.delete(user);
return new Result(true, "删除用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "删除用户失败");
}
}
@GetMapping("/show/{userName}")
@ApiOperation("查看用户")
public Result show(@PathVariable(value = "userName") String userName) {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
User user = userManagerFactory.getUserByName(userName);
FtpUser ftpUser = new FtpUser(user.getHomeDirectory(), user.getPassword(), user.getEnabled(), user.getName(), user.getMaxIdleTime(), HdfsUserManager.getUserRenamePush(userName));
return new Result(true, ftpUser, "获取用户信息成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "获取用户信息失败");
}
}
@GetMapping("/list")
@ApiOperation("列举用户")
public Result list() {
try {
HdfsUserManager userManagerFactory = new HdfsUserManager();
userManagerFactory.setFile(new File(System.getProperty("user.dir") + File.separator + "users.properties"));
String[] allUserNames = userManagerFactory.getAllUserNames();
return new Result(true, allUserNames, "列举用户成功");
}catch (Exception e) {
log.error(e);
return new Result(false, "列举用户失败");
}
}
}

View file

@ -0,0 +1,71 @@
package org.apache.hadoop.seaweed.ftp.controller.vo;
public class FtpUser {
private String homeDirectory;
private String password;
private boolean enabled;
private String name;
private int maxIdleTime;
private boolean renamePush;
public FtpUser() {
}
public FtpUser(String homeDirectory, String password, boolean enabled, String name, int maxIdleTime, boolean renamePush) {
this.homeDirectory = homeDirectory;
this.password = password;
this.enabled = enabled;
this.name = name;
this.maxIdleTime = maxIdleTime;
this.renamePush = renamePush;
}
public String getHomeDirectory() {
return homeDirectory;
}
public void setHomeDirectory(String homeDirectory) {
this.homeDirectory = homeDirectory;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getMaxIdleTime() {
return maxIdleTime;
}
public void setMaxIdleTime(int maxIdleTime) {
this.maxIdleTime = maxIdleTime;
}
public boolean isRenamePush() {
return renamePush;
}
public void setRenamePush(boolean renamePush) {
this.renamePush = renamePush;
}
}

View file

@ -0,0 +1,43 @@
package org.apache.hadoop.seaweed.ftp.controller.vo;
public class Result {
private boolean status;
private Object data;
private String message;
public Result(boolean status, String message) {
this.status = status;
this.message = message;
}
public Result(boolean status, Object data, String message) {
this.status = status;
this.message = message;
this.data = data;
}
public boolean isStatus() {
return status;
}
public void setStatus(boolean status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public Object getData() {
return data;
}
public void setData(Object data) {
this.data = data;
}
}

View file

@ -0,0 +1,102 @@
package org.apache.hadoop.seaweed.ftp.service;
import org.apache.ftpserver.DataConnectionConfiguration;
import org.apache.ftpserver.DataConnectionConfigurationFactory;
import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;
import org.apache.ftpserver.command.CommandFactoryFactory;
import org.apache.ftpserver.listener.ListenerFactory;
import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsFileSystemManager;
import org.apache.hadoop.seaweed.ftp.service.filesystem.HdfsOverFtpSystem;
import org.apache.hadoop.seaweed.ftp.users.HdfsUserManager;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.File;
/**
* reference: https://github.com/iponweb/hdfs-over-ftp
*/
@Component
public class HFtpService {
private static Logger log = Logger.getLogger(HFtpService.class);
@Value("${ftp.port}")
private int port = 0;
@Value("${ftp.passive-address}")
private String passiveAddress;
@Value("${ftp.passive-ports}")
private String passivePorts;
@Value("${hdfs.uri}")
private String hdfsUri;
@Value("${seaweedFs.enable}")
private boolean seaweedFsEnable;
@Value("${seaweedFs.access}")
private String seaweedFsAccess;
@Value("${seaweedFs.replication}")
private String seaweedFsReplication;
private FtpServer ftpServer = null;
public void startServer() throws Exception {
log.info("Starting HDFS-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
HdfsOverFtpSystem.setHdfsUri(hdfsUri);
HdfsOverFtpSystem.setSeaweedFsEnable(seaweedFsEnable);
HdfsOverFtpSystem.setSeaweedFsAccess(seaweedFsAccess);
HdfsOverFtpSystem.setSeaweedFsReplication(seaweedFsReplication);
FtpServerFactory server = new FtpServerFactory();
server.setFileSystem(new HdfsFileSystemManager());
ListenerFactory factory = new ListenerFactory();
factory.setPort(port);
DataConnectionConfigurationFactory dccFactory = new DataConnectionConfigurationFactory();
dccFactory.setPassiveAddress("0.0.0.0");
dccFactory.setPassivePorts(passivePorts);
dccFactory.setPassiveExternalAddress(passiveAddress);
DataConnectionConfiguration dcc = dccFactory.createDataConnectionConfiguration();
factory.setDataConnectionConfiguration(dcc);
server.addListener("default", factory.createListener());
HdfsUserManager userManager = new HdfsUserManager();
final File file = loadResource("/users.properties");
userManager.setFile(file);
server.setUserManager(userManager);
CommandFactoryFactory cmFact = new CommandFactoryFactory();
cmFact.setUseDefaultCommands(true);
server.setCommandFactory(cmFact.createCommandFactory());
// start the server
ftpServer = server.createServer();
ftpServer.start();
}
public void stopServer() {
log.info("Stopping Hdfs-Over-Ftp server. port: " + port + " passive-address: " + passiveAddress + " passive-ports: " + passivePorts + " hdfs-uri: " + hdfsUri);
ftpServer.stop();
}
public boolean statusServer() {
try {
return !ftpServer.isStopped();
}catch (Exception e) {
return false;
}
}
private static File loadResource(String resourceName) {
return new File(System.getProperty("user.dir") + resourceName);
}
}

View file

@ -0,0 +1,333 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FtpFile;
import org.apache.ftpserver.ftplet.User;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.seaweed.ftp.users.HdfsUser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
* This class implements all actions to HDFS
*/
public class HdfsFileObject implements FtpFile {
private final Logger log = LoggerFactory.getLogger(HdfsFileObject.class);
private Path homePath;
private Path path;
private Path fullPath;
private HdfsUser user;
/**
* Constructs HdfsFileObject from path
*
* @param path path to represent object
* @param user accessor of the object
*/
public HdfsFileObject(String homePath, String path, User user) {
this.homePath = new Path(homePath);
this.path = new Path(path);
this.fullPath = new Path(homePath + path);
this.user = (HdfsUser) user;
}
public String getAbsolutePath() {
// strip the last '/' if necessary
String fullName = path.toString();
int filelen = fullName.length();
if ((filelen != 1) && (fullName.charAt(filelen - 1) == '/')) {
fullName = fullName.substring(0, filelen - 1);
}
return fullName;
}
public String getName() {
return path.getName();
}
/**
* HDFS has no hidden objects
*
* @return always false
*/
public boolean isHidden() {
return false;
}
/**
* Checks if the object is a directory
*
* @return true if the object is a directory
*/
public boolean isDirectory() {
try {
log.debug("is directory? : " + fullPath);
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
return fs.isDir();
} catch (IOException e) {
log.debug(fullPath + " is not dir", e);
return false;
}
}
/**
* Checks if the object is a file
*
* @return true if the object is a file
*/
public boolean isFile() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
return dfs.isFile(fullPath);
} catch (IOException e) {
log.debug(fullPath + " is not file", e);
return false;
}
}
/**
* Checks if the object does exist
*
* @return true if the object does exist
*/
public boolean doesExist() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.getFileStatus(fullPath);
return true;
} catch (IOException e) {
// log.debug(path + " does not exist", e);
return false;
}
}
public boolean isReadable() {
return true;
}
public boolean isWritable() {
return true;
}
public boolean isRemovable() {
return true;
}
/**
* Get owner of the object
*
* @return owner of the object
*/
public String getOwnerName() {
return "root";
/*
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
String owner = fs.getOwner();
if(owner.length() == 0) {
return "root";
}
return owner;
} catch (IOException e) {
e.printStackTrace();
return null;
}
*/
}
/**
* Get group of the object
*
* @return group of the object
*/
public String getGroupName() {
return "root";
/*
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
String group = fs.getGroup();
if(group.length() == 0) {
return "root";
}
return group;
} catch (IOException e) {
e.printStackTrace();
return null;
}
*/
}
/**
* Get link count
*
* @return 3 is for a directory and 1 is for a file
*/
public int getLinkCount() {
return isDirectory() ? 3 : 1;
}
/**
* Get last modification date
*
* @return last modification date as a long
*/
public long getLastModified() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
return fs.getModificationTime();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
}
public boolean setLastModified(long l) {
return false;
}
/**
* Get a size of the object
*
* @return size of the object in bytes
*/
public long getSize() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fs = dfs.getFileStatus(fullPath);
log.debug("getSize(): " + fullPath + " : " + fs.getLen());
return fs.getLen();
} catch (IOException e) {
e.printStackTrace();
return 0;
}
}
public Object getPhysicalFile() {
return null;
}
/**
* Create a new dir from the object
*
* @return true if dir is created
*/
public boolean mkdir() {
try {
FileSystem fs = HdfsOverFtpSystem.getDfs();
fs.mkdirs(fullPath);
// fs.setOwner(path, user.getName(), user.getMainGroup());
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* Delete object from the HDFS filesystem
*
* @return true if the object is deleted
*/
public boolean delete() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.delete(fullPath, true);
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
public boolean move(FtpFile ftpFile) {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
dfs.rename(fullPath, new Path(fullPath.getParent() + File.separator + ftpFile.getName()));
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* List files of the directory
*
* @return List of files in the directory
*/
public List<FtpFile> listFiles() {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FileStatus fileStats[] = dfs.listStatus(fullPath);
// get the virtual name of the base directory
String virtualFileStr = getAbsolutePath();
if (virtualFileStr.charAt(virtualFileStr.length() - 1) != '/') {
virtualFileStr += '/';
}
FtpFile[] virtualFiles = new FtpFile[fileStats.length];
for (int i = 0; i < fileStats.length; i++) {
File fileObj = new File(fileStats[i].getPath().toString());
String fileName = virtualFileStr + fileObj.getName();
virtualFiles[i] = new HdfsFileObject(homePath.toString(), fileName, user);
}
return Collections.unmodifiableList(Arrays.asList(virtualFiles));
} catch (IOException e) {
log.debug("", e);
return null;
}
}
/**
* Creates output stream to write to the object
*
* @param l is not used here
* @return OutputStream
* @throws IOException
*/
public OutputStream createOutputStream(long l) {
try {
FileSystem fs = HdfsOverFtpSystem.getDfs();
FSDataOutputStream out = fs.create(fullPath);
// fs.setOwner(fullPath, user.getName(), user.getMainGroup());
return out;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
/**
* Creates input stream to read from the object
*
* @param l is not used here
* @return OutputStream
* @throws IOException
*/
public InputStream createInputStream(long l) {
try {
FileSystem dfs = HdfsOverFtpSystem.getDfs();
FSDataInputStream in = dfs.open(fullPath);
return in;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
}

View file

@ -0,0 +1,14 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FileSystemFactory;
import org.apache.ftpserver.ftplet.FileSystemView;
import org.apache.ftpserver.ftplet.User;
/**
* Impelented FileSystemManager to use HdfsFileSystemView
*/
public class HdfsFileSystemManager implements FileSystemFactory {
public FileSystemView createFileSystemView(User user) {
return new HdfsFileSystemView(user);
}
}

View file

@ -0,0 +1,104 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.ftpserver.ftplet.FileSystemView;
import org.apache.ftpserver.ftplet.FtpFile;
import org.apache.ftpserver.ftplet.User;
import org.apache.hadoop.fs.Path;
import java.io.File;
/**
* Implemented FileSystemView to use HdfsFileObject
*/
public class HdfsFileSystemView implements FileSystemView {
private String homePath;
private String currPath = File.separator;
private User user;
/**
* Constructor - set the user object.
*/
protected HdfsFileSystemView(User user) {
if (user == null) {
throw new IllegalArgumentException("user can not be null");
}
if (user.getHomeDirectory() == null) {
throw new IllegalArgumentException(
"User home directory can not be null");
}
this.homePath = user.getHomeDirectory();
this.user = user;
}
public FtpFile getHomeDirectory() {
return new HdfsFileObject(homePath, File.separator, user);
}
public FtpFile getWorkingDirectory() {
FtpFile fileObj;
if (currPath.equals(File.separator)) {
fileObj = new HdfsFileObject(homePath, File.separator, user);
} else {
fileObj = new HdfsFileObject(homePath, currPath, user);
}
return fileObj;
}
public boolean changeWorkingDirectory(String dir) {
Path path;
if (dir.startsWith(File.separator) || new Path(currPath).equals(new Path(dir))) {
path = new Path(dir);
} else if (currPath.length() > 1) {
path = new Path(currPath + File.separator + dir);
} else {
if(dir.startsWith("/")) {
path = new Path(dir);
}else {
path = new Path(File.separator + dir);
}
}
// 防止退回根目录
if (path.getName().equals("..")) {
path = new Path(File.separator);
}
HdfsFileObject file = new HdfsFileObject(homePath, path.toString(), user);
if (file.isDirectory()) {
currPath = path.toString();
return true;
} else {
return false;
}
}
public FtpFile getFile(String file) {
String path;
if (file.startsWith(File.separator)) {
path = file;
} else if (currPath.length() > 1) {
path = currPath + File.separator + file;
} else {
path = File.separator + file;
}
return new HdfsFileObject(homePath, path, user);
}
/**
* Is the file content random accessible?
*/
public boolean isRandomAccessible() {
return true;
}
/**
* Dispose file system view - does nothing.
*/
public void dispose() {
}
}

View file

@ -0,0 +1,72 @@
package org.apache.hadoop.seaweed.ftp.service.filesystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
* Class to store DFS connection
*/
public class HdfsOverFtpSystem {
private static FileSystem fs = null;
private static String hdfsUri;
private static boolean seaweedFsEnable;
private static String seaweedFsAccess;
private static String seaweedFsReplication;
private final static Logger log = LoggerFactory.getLogger(HdfsOverFtpSystem.class);
private static void hdfsInit() throws IOException {
Configuration configuration = new Configuration();
configuration.set("fs.defaultFS", hdfsUri);
if(seaweedFsEnable) {
configuration.set("fs.seaweedfs.impl", "seaweed.hdfs.SeaweedFileSystem");
configuration.set("fs.seaweed.volume.server.access", seaweedFsAccess);
configuration.set("fs.seaweed.replication", seaweedFsReplication);
}
fs = FileSystem.get(configuration);
log.info("HDFS load success");
}
/**
* Get dfs
*
* @return dfs
* @throws IOException
*/
public static FileSystem getDfs() throws IOException {
if (fs == null) {
hdfsInit();
}
return fs;
}
public static void setHdfsUri(String hdfsUri) {
HdfsOverFtpSystem.hdfsUri = hdfsUri;
}
public static String getHdfsUri() {
return hdfsUri;
}
public static void setSeaweedFsEnable(boolean seaweedFsEnable) {
HdfsOverFtpSystem.seaweedFsEnable = seaweedFsEnable;
}
public static void setSeaweedFsAccess(String seaweedFsAccess) {
HdfsOverFtpSystem.seaweedFsAccess = seaweedFsAccess;
}
public static void setSeaweedFsReplication(String seaweedFsReplication) {
HdfsOverFtpSystem.seaweedFsReplication = seaweedFsReplication;
}
}

View file

@ -0,0 +1,239 @@
package org.apache.hadoop.seaweed.ftp.users;
import org.apache.ftpserver.ftplet.Authority;
import org.apache.ftpserver.ftplet.AuthorizationRequest;
import org.apache.ftpserver.ftplet.User;
import org.apache.log4j.Logger;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class HdfsUser implements User, Serializable {
private static final long serialVersionUID = -47371353779731294L;
private String name = null;
private String password = null;
private int maxIdleTimeSec = 0; // no limit
private String homeDir = null;
private boolean isEnabled = true;
private List<? extends Authority> authorities = new ArrayList<Authority>();
private ArrayList<String> groups = new ArrayList<String>();
private Logger log = Logger.getLogger(HdfsUser.class);
/**
* Default constructor.
*/
public HdfsUser() {
}
/**
* Copy constructor.
*/
public HdfsUser(User user) {
name = user.getName();
password = user.getPassword();
authorities = user.getAuthorities();
maxIdleTimeSec = user.getMaxIdleTime();
homeDir = user.getHomeDirectory();
isEnabled = user.getEnabled();
}
public ArrayList<String> getGroups() {
return groups;
}
/**
* Get the main group of the user
*
* @return main group of the user
*/
public String getMainGroup() {
if (groups.size() > 0) {
return groups.get(0);
} else {
log.error("User " + name + " is not a memer of any group");
return "error";
}
}
/**
* Checks if user is a member of the group
*
* @param group to check
* @return true if the user id a member of the group
*/
public boolean isGroupMember(String group) {
for (String userGroup : groups) {
if (userGroup.equals(group)) {
return true;
}
}
return false;
}
/**
* Set users' groups
*
* @param groups to set
*/
public void setGroups(ArrayList<String> groups) {
if (groups.size() < 1) {
log.error("User " + name + " is not a memer of any group");
}
this.groups = groups;
}
/**
* Get the user name.
*/
public String getName() {
return name;
}
/**
* Set user name.
*/
public void setName(String name) {
this.name = name;
}
/**
* Get the user password.
*/
public String getPassword() {
return password;
}
/**
* Set user password.
*/
public void setPassword(String pass) {
password = pass;
}
public List<Authority> getAuthorities() {
if (authorities != null) {
return Collections.unmodifiableList(authorities);
} else {
return null;
}
}
public void setAuthorities(List<Authority> authorities) {
if (authorities != null) {
this.authorities = Collections.unmodifiableList(authorities);
} else {
this.authorities = null;
}
}
/**
* Get the maximum idle time in second.
*/
public int getMaxIdleTime() {
return maxIdleTimeSec;
}
/**
* Set the maximum idle time in second.
*/
public void setMaxIdleTime(int idleSec) {
maxIdleTimeSec = idleSec;
if (maxIdleTimeSec < 0) {
maxIdleTimeSec = 0;
}
}
/**
* Get the user enable status.
*/
public boolean getEnabled() {
return isEnabled;
}
/**
* Set the user enable status.
*/
public void setEnabled(boolean enb) {
isEnabled = enb;
}
/**
* Get the user home directory.
*/
public String getHomeDirectory() {
return homeDir;
}
/**
* Set the user home directory.
*/
public void setHomeDirectory(String home) {
homeDir = home;
}
/**
* String representation.
*/
public String toString() {
return name;
}
/**
* {@inheritDoc}
*/
public AuthorizationRequest authorize(AuthorizationRequest request) {
List<Authority> authorities = getAuthorities();
// check for no authorities at all
if (authorities == null) {
return null;
}
boolean someoneCouldAuthorize = false;
for (Authority authority : authorities) {
if (authority.canAuthorize(request)) {
someoneCouldAuthorize = true;
request = authority.authorize(request);
// authorization failed, return null
if (request == null) {
return null;
}
}
}
if (someoneCouldAuthorize) {
return request;
} else {
return null;
}
}
/**
* {@inheritDoc}
*/
public List<Authority> getAuthorities(Class<? extends Authority> clazz) {
List<Authority> selected = new ArrayList<Authority>();
for (Authority authority : authorities) {
if (authority.getClass().equals(clazz)) {
selected.add(authority);
}
}
return selected;
}
}

View file

@ -0,0 +1,453 @@
package org.apache.hadoop.seaweed.ftp.users;
import org.apache.ftpserver.FtpServerConfigurationException;
import org.apache.ftpserver.ftplet.*;
import org.apache.ftpserver.usermanager.*;
import org.apache.ftpserver.usermanager.impl.*;
import org.apache.ftpserver.util.BaseProperties;
import org.apache.ftpserver.util.IoUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
public class HdfsUserManager extends AbstractUserManager {
private final Logger LOG = LoggerFactory
.getLogger(HdfsUserManager.class);
private final static String DEPRECATED_PREFIX = "FtpServer.user.";
private final static String PREFIX = "ftpserver.user.";
private static BaseProperties userDataProp;
private File userDataFile = new File("users.conf");
private boolean isConfigured = false;
private PasswordEncryptor passwordEncryptor = new Md5PasswordEncryptor();
/**
* Retrieve the file used to load and store users
*
* @return The file
*/
public File getFile() {
return userDataFile;
}
/**
* Set the file used to store and read users. Must be set before
* {@link #configure()} is called.
*
* @param propFile A file containing users
*/
public void setFile(File propFile) {
if (isConfigured) {
throw new IllegalStateException("Must be called before configure()");
}
this.userDataFile = propFile;
}
/**
* Retrieve the password encryptor used for this user manager
*
* @return The password encryptor. Default to {@link Md5PasswordEncryptor}
* if no other has been provided
*/
public PasswordEncryptor getPasswordEncryptor() {
return passwordEncryptor;
}
/**
* Set the password encryptor to use for this user manager
*
* @param passwordEncryptor The password encryptor
*/
public void setPasswordEncryptor(PasswordEncryptor passwordEncryptor) {
this.passwordEncryptor = passwordEncryptor;
}
/**
* Lazy init the user manager
*/
private void lazyInit() {
if (!isConfigured) {
configure();
}
}
/**
* Configure user manager.
*/
public void configure() {
isConfigured = true;
try {
userDataProp = new BaseProperties();
if (userDataFile != null && userDataFile.exists()) {
FileInputStream fis = null;
try {
fis = new FileInputStream(userDataFile);
userDataProp.load(fis);
} finally {
IoUtils.close(fis);
}
}
} catch (IOException e) {
throw new FtpServerConfigurationException(
"Error loading user data file : "
+ userDataFile.getAbsolutePath(), e);
}
convertDeprecatedPropertyNames();
}
private void convertDeprecatedPropertyNames() {
Enumeration<?> keys = userDataProp.propertyNames();
boolean doSave = false;
while (keys.hasMoreElements()) {
String key = (String) keys.nextElement();
if (key.startsWith(DEPRECATED_PREFIX)) {
String newKey = PREFIX
+ key.substring(DEPRECATED_PREFIX.length());
userDataProp.setProperty(newKey, userDataProp.getProperty(key));
userDataProp.remove(key);
doSave = true;
}
}
if (doSave) {
try {
saveUserData();
} catch (FtpException e) {
throw new FtpServerConfigurationException(
"Failed to save updated user data", e);
}
}
}
public synchronized void save(User usr, boolean renamePush) throws FtpException {
lazyInit();
userDataProp.setProperty(PREFIX + usr.getName() + ".rename.push", renamePush);
save(usr);
}
/**
* Save user data. Store the properties.
*/
public synchronized void save(User usr) throws FtpException {
lazyInit();
// null value check
if (usr.getName() == null) {
throw new NullPointerException("User name is null.");
}
String thisPrefix = PREFIX + usr.getName() + '.';
// set other properties
userDataProp.setProperty(thisPrefix + ATTR_PASSWORD, getPassword(usr));
String home = usr.getHomeDirectory();
if (home == null) {
home = "/";
}
userDataProp.setProperty(thisPrefix + ATTR_HOME, home);
userDataProp.setProperty(thisPrefix + ATTR_ENABLE, usr.getEnabled());
userDataProp.setProperty(thisPrefix + ATTR_WRITE_PERM, usr
.authorize(new WriteRequest()) != null);
userDataProp.setProperty(thisPrefix + ATTR_MAX_IDLE_TIME, usr
.getMaxIdleTime());
TransferRateRequest transferRateRequest = new TransferRateRequest();
transferRateRequest = (TransferRateRequest) usr
.authorize(transferRateRequest);
if (transferRateRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_UPLOAD_RATE,
transferRateRequest.getMaxUploadRate());
userDataProp.setProperty(thisPrefix + ATTR_MAX_DOWNLOAD_RATE,
transferRateRequest.getMaxDownloadRate());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_UPLOAD_RATE);
userDataProp.remove(thisPrefix + ATTR_MAX_DOWNLOAD_RATE);
}
// request that always will succeed
ConcurrentLoginRequest concurrentLoginRequest = new ConcurrentLoginRequest(
0, 0);
concurrentLoginRequest = (ConcurrentLoginRequest) usr
.authorize(concurrentLoginRequest);
if (concurrentLoginRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_NUMBER,
concurrentLoginRequest.getMaxConcurrentLogins());
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_PER_IP,
concurrentLoginRequest.getMaxConcurrentLoginsPerIP());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_NUMBER);
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_PER_IP);
}
saveUserData();
}
/**
* @throws FtpException
*/
private void saveUserData() throws FtpException {
File dir = userDataFile.getAbsoluteFile().getParentFile();
if (dir != null && !dir.exists() && !dir.mkdirs()) {
String dirName = dir.getAbsolutePath();
throw new FtpServerConfigurationException(
"Cannot create directory for user data file : " + dirName);
}
// save user data
FileOutputStream fos = null;
try {
fos = new FileOutputStream(userDataFile);
userDataProp.store(fos, "Generated file - don't edit (please)");
} catch (IOException ex) {
LOG.error("Failed saving user data", ex);
throw new FtpException("Failed saving user data", ex);
} finally {
IoUtils.close(fos);
}
}
public synchronized void list() throws FtpException {
lazyInit();
Map dataMap = new HashMap();
Enumeration<String> propNames = (Enumeration<String>) userDataProp.propertyNames();
ArrayList<String> a = Collections.list(propNames);
a.remove("i18nMap");//去除i18nMap
for(String attrName : a){
// dataMap.put(attrName, propNames.);
}
}
/**
* Delete an user. Removes all this user entries from the properties. After
* removing the corresponding from the properties, save the data.
*/
public synchronized void delete(String usrName) throws FtpException {
lazyInit();
// remove entries from properties
String thisPrefix = PREFIX + usrName + '.';
Enumeration<?> propNames = userDataProp.propertyNames();
ArrayList<String> remKeys = new ArrayList<String>();
while (propNames.hasMoreElements()) {
String thisKey = propNames.nextElement().toString();
if (thisKey.startsWith(thisPrefix)) {
remKeys.add(thisKey);
}
}
Iterator<String> remKeysIt = remKeys.iterator();
while (remKeysIt.hasNext()) {
userDataProp.remove(remKeysIt.next());
}
saveUserData();
}
/**
* Get user password. Returns the encrypted value.
* <p/>
* <pre>
* If the password value is not null
* password = new password
* else
* if user does exist
* password = old password
* else
* password = &quot;&quot;
* </pre>
*/
private String getPassword(User usr) {
String name = usr.getName();
String password = usr.getPassword();
if (password != null) {
password = passwordEncryptor.encrypt(password);
} else {
String blankPassword = passwordEncryptor.encrypt("");
if (doesExist(name)) {
String key = PREFIX + name + '.' + ATTR_PASSWORD;
password = userDataProp.getProperty(key, blankPassword);
} else {
password = blankPassword;
}
}
return password;
}
/**
* Get all user names.
*/
public synchronized String[] getAllUserNames() {
lazyInit();
// get all user names
String suffix = '.' + ATTR_HOME;
ArrayList<String> ulst = new ArrayList<String>();
Enumeration<?> allKeys = userDataProp.propertyNames();
int prefixlen = PREFIX.length();
int suffixlen = suffix.length();
while (allKeys.hasMoreElements()) {
String key = (String) allKeys.nextElement();
if (key.endsWith(suffix)) {
String name = key.substring(prefixlen);
int endIndex = name.length() - suffixlen;
name = name.substring(0, endIndex);
ulst.add(name);
}
}
Collections.sort(ulst);
return ulst.toArray(new String[0]);
}
private ArrayList<String> parseGroups(String groupsLine) {
String groupsArray[] = groupsLine.split(",");
return new ArrayList(Arrays.asList(groupsArray));
}
public static synchronized boolean getUserRenamePush(String userName) {
return userDataProp.getBoolean(PREFIX + userName + ".rename.push", false);
}
/**
* Load user data.
*/
public synchronized User getUserByName(String userName) {
lazyInit();
if (!doesExist(userName)) {
return null;
}
String baseKey = PREFIX + userName + '.';
HdfsUser user = new HdfsUser();
user.setName(userName);
user.setEnabled(userDataProp.getBoolean(baseKey + ATTR_ENABLE, true));
user.setHomeDirectory(userDataProp
.getProperty(baseKey + ATTR_HOME, "/"));
// user.setGroups(parseGroups(userDataProp
// .getProperty(baseKey + "groups")));
List<Authority> authorities = new ArrayList<Authority>();
if (userDataProp.getBoolean(baseKey + ATTR_WRITE_PERM, false)) {
authorities.add(new WritePermission());
}
int maxLogin = userDataProp.getInteger(baseKey + ATTR_MAX_LOGIN_NUMBER,
0);
int maxLoginPerIP = userDataProp.getInteger(baseKey
+ ATTR_MAX_LOGIN_PER_IP, 0);
authorities.add(new ConcurrentLoginPermission(maxLogin, maxLoginPerIP));
int uploadRate = userDataProp.getInteger(
baseKey + ATTR_MAX_UPLOAD_RATE, 0);
int downloadRate = userDataProp.getInteger(baseKey
+ ATTR_MAX_DOWNLOAD_RATE, 0);
authorities.add(new TransferRatePermission(downloadRate, uploadRate));
user.setAuthorities(authorities);
user.setMaxIdleTime(userDataProp.getInteger(baseKey
+ ATTR_MAX_IDLE_TIME, 0));
return user;
}
/**
* User existance check
*/
public synchronized boolean doesExist(String name) {
lazyInit();
String key = PREFIX + name + '.' + ATTR_HOME;
return userDataProp.containsKey(key);
}
/**
* User authenticate method
*/
public synchronized User authenticate(Authentication authentication)
throws AuthenticationFailedException {
lazyInit();
if (authentication instanceof UsernamePasswordAuthentication) {
UsernamePasswordAuthentication upauth = (UsernamePasswordAuthentication) authentication;
String user = upauth.getUsername();
String password = upauth.getPassword();
if (user == null) {
throw new AuthenticationFailedException("Authentication failed");
}
if (password == null) {
password = "";
}
String storedPassword = userDataProp.getProperty(PREFIX + user + '.'
+ ATTR_PASSWORD);
if (storedPassword == null) {
// user does not exist
throw new AuthenticationFailedException("Authentication failed");
}
if (passwordEncryptor.matches(password, storedPassword)) {
return getUserByName(user);
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else if (authentication instanceof AnonymousAuthentication) {
if (doesExist("anonymous")) {
return getUserByName("anonymous");
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else {
throw new IllegalArgumentException(
"Authentication not supported by this user manager");
}
}
/**
* Close the user manager - remove existing entries.
*/
public synchronized void dispose() {
if (userDataProp != null) {
userDataProp.clear();
userDataProp = null;
}
}
}

View file

@ -0,0 +1,15 @@
server:
port: 8080
ftp:
port: 2222
passive-address: localhost
passive-ports: 30000-30999
hdfs:
uri: seaweedfs://localhost:8888
seaweedFs:
enable: true
access: direct # direct/filerProxy/publicUrl
replication: "000"

View file

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>package</id>
<formats>
<!-- 指定打包格式支持的打包格式有zip、tar、tar.gz (or tgz)、tar.bz2 (or tbz2)、jar、dir、war可以同时指定多个打包格式 -->
<format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
<directory>src/main/resources</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>application.yml</include>
<include>logback-spring.xml</include>
<include>users.properties</include>
<include>kafka-producer.properties</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>lib</outputDirectory>
<scope>runtime</scope>
<unpack>false</unpack>
</dependencySet>
</dependencySets>
</assembly>

View file

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
<!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径-->
<property name="LOG_HOME" value="${user.dir}/logs/" />
<!-- 控制台输出 -->
<appender name="Stdout" class="ch.qos.logback.core.ConsoleAppender">
<!-- 日志输出编码 -->
<layout class="ch.qos.logback.classic.PatternLayout">
<!--格式化输出:%d表示日期%thread表示线程名%-5level级别从左显示5个字符宽度%msg日志消息%n是换行符-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n
</pattern>
</layout>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="RollingFile"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>${LOG_HOME}/fileLog.log</File>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_HOME}/fileLog.log.%d.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>100 MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
</rollingPolicy>
<encoder>
<pattern>
%d %p (%file:%line\)- %m%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- 日志输出级别 -->
<root level="info">
<appender-ref ref="Stdout" />
<appender-ref ref="RollingFile" />
</root>
</configuration>

View file

@ -0,0 +1,12 @@
#Generated file - don't edit (please)
#Thu Mar 11 19:11:12 CST 2021
ftpserver.user.test.idletime=0
ftpserver.user.test.maxloginperip=0
ftpserver.user.test.userpassword=44664D4D827C740293D2AA244FB60445
ftpserver.user.test.enableflag=true
ftpserver.user.test.maxloginnumber=0
ftpserver.user.test.rename.push=true
ftpserver.user.test.homedirectory=/buckets/test/
ftpserver.user.test.downloadrate=0
ftpserver.user.test.writepermission=true
ftpserver.user.test.uploadrate=0

View file

@ -301,7 +301,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.6.2</seaweedfs.client.version> <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.6.2</seaweedfs.client.version> <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>

View file

@ -309,7 +309,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.6.2</seaweedfs.client.version> <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.6.2</seaweedfs.client.version> <seaweedfs.client.version>1.6.4</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>

View file

@ -8,7 +8,7 @@
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
<properties> <properties>
<guava.version>28.0-jre</guava.version> <guava.version>30.0-jre</guava.version>
</properties> </properties>
<dependencies> <dependencies>

View file

@ -41,6 +41,7 @@ type BenchmarkOptions struct {
grpcDialOption grpc.DialOption grpcDialOption grpc.DialOption
masterClient *wdclient.MasterClient masterClient *wdclient.MasterClient
fsync *bool fsync *bool
useTcp *bool
} }
var ( var (
@ -67,6 +68,7 @@ func init() {
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
b.fsync = cmdBenchmark.Flag.Bool("fsync", false, "flush data to disk after write") b.fsync = cmdBenchmark.Flag.Bool("fsync", false, "flush data to disk after write")
b.useTcp = cmdBenchmark.Flag.Bool("useTcp", false, "send data via tcp")
sharedBytes = make([]byte, 1024) sharedBytes = make([]byte, 1024)
} }
@ -223,6 +225,8 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
random := rand.New(rand.NewSource(time.Now().UnixNano())) random := rand.New(rand.NewSource(time.Now().UnixNano()))
volumeTcpClient := wdclient.NewVolumeTcpClient()
for id := range idChan { for id := range idChan {
start := time.Now() start := time.Now()
fileSize := int64(*b.fileSize + random.Intn(64)) fileSize := int64(*b.fileSize + random.Intn(64))
@ -243,7 +247,15 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
if !isSecure && assignResult.Auth != "" { if !isSecure && assignResult.Auth != "" {
isSecure = true isSecure = true
} }
if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil { if *b.useTcp {
if uploadByTcp(volumeTcpClient, fp) {
fileIdLineChan <- fp.Fid
s.completed++
s.transferred += fileSize
} else {
s.failed++
}
} else if _, err := fp.Upload(0, b.masterClient.GetMaster, false, assignResult.Auth, b.grpcDialOption); err == nil {
if random.Intn(100) < *b.deletePercentage { if random.Intn(100) < *b.deletePercentage {
s.total++ s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
@ -293,7 +305,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
} }
var bytes []byte var bytes []byte
for _, url := range urls { for _, url := range urls {
bytes, _, err = util.FastGet(url) bytes, _, err = util.Get(url)
if err == nil { if err == nil {
break break
} }
@ -329,6 +341,17 @@ func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan b
} }
} }
func uploadByTcp(volumeTcpClient *wdclient.VolumeTcpClient, fp *operation.FilePart) bool {
err := volumeTcpClient.PutFileChunk(fp.Server, fp.Fid, uint32(fp.FileSize), fp.Reader)
if err != nil {
glog.Errorf("upload chunk err: %v", err)
return false
}
return true
}
func readFileIds(fileName string, fileIdLineChan chan string) { func readFileIds(fileName string, fileIdLineChan chan string) {
file, err := os.Open(fileName) // For read access. file, err := os.Open(fileName) // For read access.
if err != nil { if err != nil {

View file

@ -17,13 +17,16 @@ var Commands = []*Command{
cmdFiler, cmdFiler,
cmdFilerBackup, cmdFilerBackup,
cmdFilerCat, cmdFilerCat,
cmdFilerMetaBackup,
cmdFilerMetaTail, cmdFilerMetaTail,
cmdFilerReplicate, cmdFilerReplicate,
cmdFilerSynchronize, cmdFilerSynchronize,
cmdFix, cmdFix,
cmdGateway,
cmdMaster, cmdMaster,
cmdMount, cmdMount,
cmdS3, cmdS3,
cmdIam,
cmdMsgBroker, cmdMsgBroker,
cmdScaffold, cmdScaffold,
cmdServer, cmdServer,

View file

@ -215,7 +215,7 @@ func runExport(cmd *Command, args []string) bool {
err = storage.ScanVolumeFile(util.ResolvePath(*export.dir), *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner) err = storage.ScanVolumeFile(util.ResolvePath(*export.dir), *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
glog.Fatalf("Export Volume File [ERROR] %s\n", err) glog.Errorf("Export Volume File [ERROR] %s\n", err)
} }
return true return true
} }

View file

@ -25,6 +25,8 @@ var (
filerS3Options S3Options filerS3Options S3Options
filerStartWebDav *bool filerStartWebDav *bool
filerWebDavOptions WebDavOption filerWebDavOptions WebDavOption
filerStartIam *bool
filerIamOptions IamOptions
) )
type FilerOptions struct { type FilerOptions struct {
@ -47,6 +49,7 @@ type FilerOptions struct {
metricsHttpPort *int metricsHttpPort *int
saveToFilerLimit *int saveToFilerLimit *int
defaultLevelDbDirectory *string defaultLevelDbDirectory *string
concurrentUploadLimitMB *int
} }
func init() { func init() {
@ -54,12 +57,12 @@ func init() {
f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection") f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address") f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public") f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.") f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit") f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size") f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center") f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack") f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
@ -69,6 +72,7 @@ func init() {
f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store") f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory") f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
// start s3 on filer // start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@ -89,6 +93,10 @@ func init() {
filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
// start iam on filer
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
} }
var cmdFiler = &Command{ var cmdFiler = &Command{
@ -106,7 +114,7 @@ var cmdFiler = &Command{
GET /path/to/ GET /path/to/
The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order. The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
If the "filer.toml" is not found, an embedded filer store will be craeted under "-defaultStoreDir". If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".
The example filer.toml configuration file can be generated by "weed scaffold -config=filer" The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
@ -119,22 +127,33 @@ func runFiler(cmd *Command, args []string) bool {
go stats_collect.StartMetricsServer(*f.metricsHttpPort) go stats_collect.StartMetricsServer(*f.metricsHttpPort)
filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
startDelay := time.Duration(2)
if *filerStartS3 { if *filerStartS3 {
filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
filerS3Options.filer = &filerAddress filerS3Options.filer = &filerAddress
go func() { go func() {
time.Sleep(2 * time.Second) time.Sleep(startDelay * time.Second)
filerS3Options.startS3Server() filerS3Options.startS3Server()
}() }()
startDelay++
} }
if *filerStartWebDav { if *filerStartWebDav {
filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
filerWebDavOptions.filer = &filerAddress filerWebDavOptions.filer = &filerAddress
go func() { go func() {
time.Sleep(2 * time.Second) time.Sleep(startDelay * time.Second)
filerWebDavOptions.startWebDav() filerWebDavOptions.startWebDav()
}() }()
startDelay++
}
if *filerStartIam {
filerIamOptions.filer = &filerAddress
filerIamOptions.masters = f.masters
go func() {
time.Sleep(startDelay * time.Second)
filerIamOptions.startIamServer()
}()
} }
f.startFiler() f.startFiler()
@ -159,21 +178,22 @@ func (fo *FilerOptions) startFiler() {
} }
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{ fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
Masters: strings.Split(*fo.masters, ","), Masters: strings.Split(*fo.masters, ","),
Collection: *fo.collection, Collection: *fo.collection,
DefaultReplication: *fo.defaultReplicaPlacement, DefaultReplication: *fo.defaultReplicaPlacement,
DisableDirListing: *fo.disableDirListing, DisableDirListing: *fo.disableDirListing,
MaxMB: *fo.maxMB, MaxMB: *fo.maxMB,
DirListingLimit: *fo.dirListingLimit, DirListingLimit: *fo.dirListingLimit,
DataCenter: *fo.dataCenter, DataCenter: *fo.dataCenter,
Rack: *fo.rack, Rack: *fo.rack,
DefaultLevelDbDir: defaultLevelDbDirectory, DefaultLevelDbDir: defaultLevelDbDirectory,
DisableHttp: *fo.disableHttp, DisableHttp: *fo.disableHttp,
Host: *fo.ip, Host: *fo.ip,
Port: uint32(*fo.port), Port: uint32(*fo.port),
Cipher: *fo.cipher, Cipher: *fo.cipher,
SaveToFilerLimit: *fo.saveToFilerLimit, SaveToFilerLimit: int64(*fo.saveToFilerLimit),
Filers: peers, Filers: peers,
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
}) })
if nfs_err != nil { if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err) glog.Fatalf("Filer startup error: %v", nfs_err)

View file

@ -155,4 +155,3 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}) })
} }

View file

@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool {
filerCat.filerClient = client filerCat.filerClient = client
return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false)
}) })

View file

@ -56,7 +56,7 @@ func init() {
copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name") copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit") copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines") copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file") copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
} }
@ -207,16 +207,6 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
} }
mode := fi.Mode() mode := fi.Mode()
if mode.IsDir() {
files, _ := ioutil.ReadDir(fileOrDir)
for _, subFileOrDir := range files {
if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
return err
}
}
return nil
}
uid, gid := util.GetFileUidGid(fi) uid, gid := util.GetFileUidGid(fi)
fileCopyTaskChan <- FileCopyTask{ fileCopyTaskChan <- FileCopyTask{
@ -228,6 +218,16 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
gid: gid, gid: gid,
} }
if mode.IsDir() {
files, _ := ioutil.ReadDir(fileOrDir)
println("checking directory", fileOrDir)
for _, subFileOrDir := range files {
if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
return err
}
}
}
return nil return nil
} }
@ -293,20 +293,22 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
// upload the file content // upload the file content
fileName := filepath.Base(f.Name()) fileName := filepath.Base(f.Name())
mimeType := detectMimeType(f) var mimeType string
data, err := ioutil.ReadAll(f)
if err != nil {
return err
}
var chunks []*filer_pb.FileChunk var chunks []*filer_pb.FileChunk
var assignResult *filer_pb.AssignVolumeResponse var assignResult *filer_pb.AssignVolumeResponse
var assignError error var assignError error
if task.fileSize > 0 { if task.fileMode & os.ModeDir == 0 && task.fileSize > 0 {
mimeType = detectMimeType(f)
data, err := ioutil.ReadAll(f)
if err != nil {
return err
}
// assign a volume // assign a volume
err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { err = pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.AssignVolumeRequest{ request := &filer_pb.AssignVolumeRequest{
Count: 1, Count: 1,

View file

@ -0,0 +1,268 @@
package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
"google.golang.org/grpc"
"io"
"reflect"
"time"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
metaBackup FilerMetaBackupOptions
)
type FilerMetaBackupOptions struct {
grpcDialOption grpc.DialOption
filerAddress *string
filerDirectory *string
restart *bool
backupFilerConfig *string
store filer.FilerStore
}
func init() {
cmdFilerMetaBackup.Run = runFilerMetaBackup // break init cycle
metaBackup.filerAddress = cmdFilerMetaBackup.Flag.String("filer", "localhost:8888", "filer hostname:port")
metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer")
metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup")
metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store")
}
var cmdFilerMetaBackup = &Command{
UsageLine: "filer.meta.backup [-filer=localhost:8888] [-filerDir=/] [-restart] -config=/path/to/backup_filer.toml",
Short: "continuously backup filer meta data changes to anther filer store specified in a backup_filer.toml",
Long: `continuously backup filer meta data changes.
The backup writes to another filer store specified in a backup_filer.toml.
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888"
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" -restart
`,
}
func runFilerMetaBackup(cmd *Command, args []string) bool {
metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
// load backup_filer.toml
v := viper.New()
v.SetConfigFile(*metaBackup.backupFilerConfig)
if err := v.ReadInConfig(); err != nil { // Handle errors reading the config file
glog.Fatalf("Failed to load %s file.\nPlease use this command to generate the a %s.toml file\n"+
" weed scaffold -config=%s -output=.\n\n\n",
*metaBackup.backupFilerConfig, "backup_filer", "filer")
}
if err := metaBackup.initStore(v); err != nil {
glog.V(0).Infof("init backup filer store: %v", err)
return true
}
missingPreviousBackup := false
_, err := metaBackup.getOffset()
if err != nil {
missingPreviousBackup = true
}
if *metaBackup.restart || missingPreviousBackup {
glog.V(0).Infof("traversing metadata tree...")
startTime := time.Now()
if err := metaBackup.traverseMetadata(); err != nil {
glog.Errorf("traverse meta data: %v", err)
return true
}
glog.V(0).Infof("metadata copied up to %v", startTime)
if err := metaBackup.setOffset(startTime); err != nil {
startTime = time.Now()
}
}
for {
err := metaBackup.streamMetadataBackup()
if err != nil {
glog.Errorf("filer meta backup from %s: %v", *metaBackup.filerAddress, err)
time.Sleep(1747 * time.Millisecond)
}
}
return true
}
func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error {
// load configuration for default filer store
hasDefaultStoreConfigured := false
for _, store := range filer.Stores {
if v.GetBool(store.GetName() + ".enabled") {
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(filer.FilerStore)
if err := store.Initialize(v, store.GetName()+"."); err != nil {
glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err)
}
glog.V(0).Infof("configured filer store to %s", store.GetName())
hasDefaultStoreConfigured = true
metaBackup.store = filer.NewFilerStoreWrapper(store)
break
}
}
if !hasDefaultStoreConfigured {
return fmt.Errorf("no filer store enabled in %s", v.ConfigFileUsed())
}
return nil
}
func (metaBackup *FilerMetaBackupOptions) traverseMetadata() (err error) {
var saveErr error
traverseErr := filer_pb.TraverseBfs(metaBackup, util.FullPath(*metaBackup.filerDirectory), func(parentPath util.FullPath, entry *filer_pb.Entry) {
println("+", parentPath.Child(entry.Name))
if err := metaBackup.store.InsertEntry(context.Background(), filer.FromPbEntry(string(parentPath), entry)); err != nil {
saveErr = fmt.Errorf("insert entry error: %v\n", err)
return
}
})
if traverseErr != nil {
return fmt.Errorf("traverse: %v", traverseErr)
}
return saveErr
}
var (
MetaBackupKey = []byte("metaBackup")
)
func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
startTime, err := metaBackup.getOffset()
if err != nil {
startTime = time.Now()
}
glog.V(0).Infof("streaming from %v", startTime)
store := metaBackup.store
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
ctx := context.Background()
message := resp.EventNotification
if message.OldEntry == nil && message.NewEntry == nil {
return nil
}
if message.OldEntry == nil && message.NewEntry != nil {
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
return store.InsertEntry(ctx, entry)
}
if message.OldEntry != nil && message.NewEntry == nil {
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name))
}
if message.OldEntry != nil && message.NewEntry != nil {
if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
return store.UpdateEntry(ctx, entry)
}
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil {
return err
}
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
return store.InsertEntry(ctx, filer.FromPbEntry(message.NewParentPath, message.NewEntry))
}
return nil
}
tailErr := pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
ClientName: "meta_backup",
PathPrefix: *metaBackup.filerDirectory,
SinceNs: startTime.UnixNano(),
})
if err != nil {
return fmt.Errorf("listen: %v", err)
}
var counter int64
var lastWriteTime time.Time
for {
resp, listenErr := stream.Recv()
if listenErr == io.EOF {
return nil
}
if listenErr != nil {
return listenErr
}
if err = eachEntryFunc(resp); err != nil {
return err
}
counter++
if lastWriteTime.Add(3 * time.Second).Before(time.Now()) {
glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, time.Unix(0, resp.TsNs), float64(counter)/float64(3))
counter = 0
lastWriteTime = time.Now()
if err2 := metaBackup.setOffset(lastWriteTime); err2 != nil {
return err2
}
}
}
})
return tailErr
}
func (metaBackup *FilerMetaBackupOptions) getOffset() (lastWriteTime time.Time, err error) {
value, err := metaBackup.store.KvGet(context.Background(), MetaBackupKey)
if err != nil {
return
}
tsNs := util.BytesToUint64(value)
return time.Unix(0, int64(tsNs)), nil
}
func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) error {
valueBuf := make([]byte, 8)
util.Uint64toBytes(valueBuf, uint64(lastWriteTime.UnixNano()))
if err := metaBackup.store.KvPut(context.Background(), MetaBackupKey, valueBuf); err != nil {
return err
}
return nil
}
var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})
func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
return fn(client)
})
}
func (metaBackup *FilerMetaBackupOptions) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}

View file

@ -23,7 +23,7 @@ func init() {
} }
var cmdFilerMetaTail = &Command{ var cmdFilerMetaTail = &Command{
UsageLine: "filer.meta.tail [-filer=localhost:8888] [-target=/]", UsageLine: "filer.meta.tail [-filer=localhost:8888] [-pathPrefix=/]",
Short: "see continuous changes on a filer", Short: "see continuous changes on a filer",
Long: `See continuous changes on a filer. Long: `See continuous changes on a filer.
@ -36,7 +36,7 @@ var cmdFilerMetaTail = &Command{
var ( var (
tailFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port") tailFiler = cmdFilerMetaTail.Flag.String("filer", "localhost:8888", "filer hostname:port")
tailTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer") tailTarget = cmdFilerMetaTail.Flag.String("pathPrefix", "/", "path to a folder or common prefix for the folders or files on filer")
tailStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"") tailStart = cmdFilerMetaTail.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
tailPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ") tailPattern = cmdFilerMetaTail.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ")
esServers = cmdFilerMetaTail.Flag.String("es", "", "comma-separated elastic servers http://<host:port>") esServers = cmdFilerMetaTail.Flag.String("es", "", "comma-separated elastic servers http://<host:port>")

93
weed/command/gateway.go Normal file
View file

@ -0,0 +1,93 @@
package command
import (
"net/http"
"strconv"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
gatewayOptions GatewayOptions
)
type GatewayOptions struct {
masters *string
filers *string
bindIp *string
port *int
maxMB *int
}
func init() {
cmdGateway.Run = runGateway // break init cycle
gatewayOptions.masters = cmdGateway.Flag.String("master", "localhost:9333", "comma-separated master servers")
gatewayOptions.filers = cmdGateway.Flag.String("filer", "localhost:8888", "comma-separated filer servers")
gatewayOptions.bindIp = cmdGateway.Flag.String("ip.bind", "localhost", "ip address to bind to")
gatewayOptions.port = cmdGateway.Flag.Int("port", 5647, "gateway http listen port")
gatewayOptions.maxMB = cmdGateway.Flag.Int("maxMB", 4, "split files larger than the limit")
}
var cmdGateway = &Command{
UsageLine: "gateway -port=8888 -master=<ip:port>[,<ip:port>]* -filer=<ip:port>[,<ip:port>]*",
Short: "start a gateway server that points to a list of master servers or a list of filers",
Long: `start a gateway server which accepts REST operation to write any blobs, files, or topic messages.
POST /blobs/
upload the blob and return a chunk id
DELETE /blobs/<chunk_id>
delete a chunk id
/*
POST /files/path/to/a/file
save /path/to/a/file on filer
DELETE /files/path/to/a/file
delete /path/to/a/file on filer
POST /topics/topicName
save on filer to /topics/topicName/<ds>/ts.json
*/
`,
}
func runGateway(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
gatewayOptions.startGateway()
return true
}
func (gw *GatewayOptions) startGateway() {
defaultMux := http.NewServeMux()
_, gws_err := weed_server.NewGatewayServer(defaultMux, &weed_server.GatewayOption{
Masters: strings.Split(*gw.masters, ","),
Filers: strings.Split(*gw.filers, ","),
MaxMB: *gw.maxMB,
})
if gws_err != nil {
glog.Fatalf("Gateway startup error: %v", gws_err)
}
glog.V(0).Infof("Start Seaweed Gateway %s at %s:%d", util.Version(), *gw.bindIp, *gw.port)
gatewayListener, e := util.NewListener(
*gw.bindIp+":"+strconv.Itoa(*gw.port),
time.Duration(10)*time.Second,
)
if e != nil {
glog.Fatalf("Filer listener error: %v", e)
}
httpS := &http.Server{Handler: defaultMux}
if err := httpS.Serve(gatewayListener); err != nil {
glog.Fatalf("Gateway Fail to serve: %v", e)
}
}

97
weed/command/iam.go Normal file
View file

@ -0,0 +1,97 @@
package command
import (
"context"
"fmt"
"net/http"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/iamapi"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux"
"time"
)
var (
iamStandaloneOptions IamOptions
)
type IamOptions struct {
filer *string
masters *string
port *int
}
func init() {
cmdIam.Run = runIam // break init cycle
iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address")
iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
}
var cmdIam = &Command{
UsageLine: "iam [-port=8111] [-filer=<ip:port>] [-masters=<ip:port>,<ip:port>]",
Short: "start a iam API compatible server",
Long: "start a iam API compatible server.",
}
func runIam(cmd *Command, args []string) bool {
return iamStandaloneOptions.startIamServer()
}
func (iamopt *IamOptions) startIamServer() bool {
filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer)
if err != nil {
glog.Fatal(err)
return false
}
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
for {
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
}
glog.V(0).Infof("IAM read filer configuration: %s", resp)
return nil
})
if err != nil {
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
time.Sleep(time.Second)
} else {
glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
break
}
}
router := mux.NewRouter().SkipClean(true)
_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
Filer: *iamopt.filer,
Port: *iamopt.port,
FilerGrpcAddress: filerGrpcAddress,
GrpcDialOption: grpcDialOption,
})
glog.V(0).Info("NewIamApiServer created")
if iamApiServer_err != nil {
glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
}
httpS := &http.Server{Handler: router}
listenAddress := fmt.Sprintf(":%d", *iamopt.port)
iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
if err != nil {
glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
}
glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
if err = httpS.Serve(iamApiListener); err != nil {
glog.Fatalf("IAM API Server Fail to serve: %v", err)
}
return true
}

View file

@ -6,7 +6,6 @@ import (
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"net/http" "net/http"
"os" "os"
"runtime"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@ -48,8 +47,8 @@ type MasterOptions struct {
func init() { func init() {
cmdMaster.Run = runMaster // break init cycle cmdMaster.Run = runMaster // break init cycle
m.port = cmdMaster.Flag.Int("port", 9333, "http listen port") m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address") m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095") m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@ -86,7 +85,6 @@ func runMaster(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false) util.LoadConfiguration("security", false)
util.LoadConfiguration("master", false) util.LoadConfiguration("master", false)
runtime.GOMAXPROCS(runtime.NumCPU())
grace.SetupProfiling(*masterCpuProfile, *masterMemProfile) grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
parent, _ := util.FullPath(*m.metaFolder).DirAndName() parent, _ := util.FullPath(*m.metaFolder).DirAndName()
@ -138,7 +136,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
if err != nil { if err != nil {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err) glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
} }
// Create your protocol servers.
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master")) grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
master_pb.RegisterSeaweedServer(grpcS, ms) master_pb.RegisterSeaweedServer(grpcS, ms)
protobuf.RegisterRaftServer(grpcS, raftServer) protobuf.RegisterRaftServer(grpcS, raftServer)

View file

@ -25,6 +25,7 @@ type MountOptions struct {
volumeServerAccess *string volumeServerAccess *string
uidMap *string uidMap *string
gidMap *string gidMap *string
readOnly *bool
} }
var ( var (
@ -45,7 +46,7 @@ func init() {
mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") mountOptions.diskType = cmdMount.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0") mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)") mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
@ -55,6 +56,7 @@ func init() {
mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") mountOptions.volumeServerAccess = cmdMount.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]")
mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>") mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>") mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file") mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file") mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

View file

@ -53,7 +53,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
filer := *option.filer filer := *option.filer
// parse filer grpc address // parse filer grpc address
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(filer) filerGrpcAddress, err := pb.ParseServerToGrpcAddress(filer)
if err != nil { if err != nil {
glog.V(0).Infof("ParseFilerGrpcAddress: %v", err) glog.V(0).Infof("ParseFilerGrpcAddress: %v", err)
return true return true
@ -63,16 +63,23 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
// try to connect to filer, filerBucketsPath may be useful later // try to connect to filer, filerBucketsPath may be useful later
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
var cipher bool var cipher bool
err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { for i := 0; i < 10; i++ {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer grpc address %s configuration: %v", filerGrpcAddress, err)
}
cipher = resp.Cipher
return nil
})
if err != nil { if err != nil {
return fmt.Errorf("get filer grpc address %s configuration: %v", filerGrpcAddress, err) glog.V(0).Infof("failed to talk to filer %s: %v", filerGrpcAddress, err)
glog.V(0).Infof("wait for %d seconds ...", i+1)
time.Sleep(time.Duration(i+1) * time.Second)
} }
cipher = resp.Cipher }
return nil
})
if err != nil { if err != nil {
glog.Infof("failed to talk to filer %s: %v", filerGrpcAddress, err) glog.Errorf("failed to talk to filer %s: %v", filerGrpcAddress, err)
return true return true
} }
@ -142,8 +149,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
fuse.Subtype("seaweedfs"), fuse.Subtype("seaweedfs"),
// fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders // fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders
fuse.NoAppleXattr(), fuse.NoAppleXattr(),
fuse.NoBrowse(),
fuse.AutoXattr(),
fuse.ExclCreate(), fuse.ExclCreate(),
fuse.DaemonTimeout("3600"), fuse.DaemonTimeout("3600"),
fuse.AllowSUID(), fuse.AllowSUID(),
@ -162,6 +167,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
if *option.nonempty { if *option.nonempty {
options = append(options, fuse.AllowNonEmptyMount()) options = append(options, fuse.AllowNonEmptyMount())
} }
if *option.readOnly {
options = append(options, fuse.ReadOnly())
}
// find mount point // find mount point
mountRoot := filerMountRootPath mountRoot := filerMountRootPath
@ -186,7 +194,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
CacheDir: *option.cacheDir, CacheDir: *option.cacheDir,
CacheSizeMB: *option.cacheSizeMB, CacheSizeMB: *option.cacheSizeMB,
DataCenter: *option.dataCenter, DataCenter: *option.dataCenter,
EntryCacheTtl: 3 * time.Second,
MountUid: uid, MountUid: uid,
MountGid: gid, MountGid: gid,
MountMode: mountMode, MountMode: mountMode,

View file

@ -63,7 +63,7 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile) grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile)
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*msgBrokerOpt.filer) filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*msgBrokerOpt.filer)
if err != nil { if err != nil {
glog.Fatal(err) glog.Fatal(err)
return false return false

View file

@ -137,7 +137,7 @@ func runS3(cmd *Command, args []string) bool {
func (s3opt *S3Options) startS3Server() bool { func (s3opt *S3Options) startS3Server() bool {
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*s3opt.filer) filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*s3opt.filer)
if err != nil { if err != nil {
glog.Fatal(err) glog.Fatal(err)
return false return false

View file

@ -103,9 +103,9 @@ dir = "./filerrdb" # directory to store rocksdb files
[mysql] # or memsql, tidb [mysql] # or memsql, tidb
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) COMMENT 'directory or file name', # name VARCHAR(1000) BINARY COMMENT 'directory or file name',
# directory TEXT COMMENT 'full path to parent directory', # directory TEXT COMMENT 'full path to parent directory',
# meta LONGBLOB, # meta LONGBLOB,
# PRIMARY KEY (dirhash, name) # PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8; # ) DEFAULT CHARSET=utf8;
@ -120,13 +120,16 @@ connection_max_idle = 2
connection_max_open = 100 connection_max_open = 100
connection_max_lifetime_seconds = 0 connection_max_lifetime_seconds = 0
interpolateParams = false interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
[mysql2] # or memsql, tidb [mysql2] # or memsql, tidb
enabled = false enabled = false
createTable = """ createTable = """
CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
dirhash BIGINT, dirhash BIGINT,
name VARCHAR(1000), name VARCHAR(1000) BINARY,
directory TEXT, directory TEXT,
meta LONGBLOB, meta LONGBLOB,
PRIMARY KEY (dirhash, name) PRIMARY KEY (dirhash, name)
@ -141,6 +144,9 @@ connection_max_idle = 2
connection_max_open = 100 connection_max_open = 100
connection_max_lifetime_seconds = 0 connection_max_lifetime_seconds = 0
interpolateParams = false interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
[postgres] # or cockroachdb, YugabyteDB [postgres] # or cockroachdb, YugabyteDB
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
@ -161,6 +167,9 @@ sslmode = "disable"
connection_max_idle = 100 connection_max_idle = 100
connection_max_open = 100 connection_max_open = 100
connection_max_lifetime_seconds = 0 connection_max_lifetime_seconds = 0
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
[postgres2] [postgres2]
enabled = false enabled = false
@ -183,6 +192,9 @@ sslmode = "disable"
connection_max_idle = 100 connection_max_idle = 100
connection_max_open = 100 connection_max_open = 100
connection_max_lifetime_seconds = 0 connection_max_lifetime_seconds = 0
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
[cassandra] [cassandra]
# CREATE TABLE filemeta ( # CREATE TABLE filemeta (
@ -269,7 +281,7 @@ index.max_result_window = 10000
# Make sure they are not the same if using the same store type! # Make sure they are not the same if using the same store type!
# 4. Set enabled to true # 4. Set enabled to true
# #
# The following is just using cassandra as an example # The following is just using redis as an example
########################## ##########################
[redis2.tmp] [redis2.tmp]
enabled = false enabled = false
@ -440,22 +452,28 @@ expires_after_seconds = 10 # seconds
# the host name is not checked, so the PERM files can be shared. # the host name is not checked, so the PERM files can be shared.
[grpc] [grpc]
ca = "" ca = ""
# Set wildcard domain for enable TLS authentication by common names
allowed_wildcard_domain = "" # .mycompany.com
[grpc.volume] [grpc.volume]
cert = "" cert = ""
key = "" key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
[grpc.master] [grpc.master]
cert = "" cert = ""
key = "" key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
[grpc.filer] [grpc.filer]
cert = "" cert = ""
key = "" key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
[grpc.msg_broker] [grpc.msg_broker]
cert = "" cert = ""
key = "" key = ""
allowed_commonNames = "" # comma-separated SSL certificate common names
# use this for any place needs a grpc client # use this for any place needs a grpc client
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload" # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
@ -463,7 +481,6 @@ key = ""
cert = "" cert = ""
key = "" key = ""
# volume server https options # volume server https options
# Note: work in progress! # Note: work in progress!
# this does not work with other clients, e.g., "weed filer|mount" etc, yet. # this does not work with other clients, e.g., "weed filer|mount" etc, yet.
@ -501,7 +518,7 @@ default = "localhost:8888" # used by maintenance scripts if the scripts needs
[master.sequencer] [master.sequencer]
type = "raft" # Choose [raft|etcd] type for storing the file id sequence type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence
# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
# example : http://127.0.0.1:2379,http://127.0.0.1:2389 # example : http://127.0.0.1:2379,http://127.0.0.1:2389
sequencer_etcd_urls = "http://127.0.0.1:2379" sequencer_etcd_urls = "http://127.0.0.1:2379"

View file

@ -2,9 +2,8 @@ package command
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"os" "os"
"runtime"
"runtime/pprof"
"strings" "strings"
"time" "time"
@ -16,6 +15,7 @@ import (
type ServerOptions struct { type ServerOptions struct {
cpuprofile *string cpuprofile *string
memprofile *string
v VolumeServerOptions v VolumeServerOptions
} }
@ -49,8 +49,8 @@ var cmdServer = &Command{
} }
var ( var (
serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name") serverIp = cmdServer.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") serverBindIp = cmdServer.Flag.String("ip.bind", "", "ip address to bind to")
serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds") serverTimeout = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
@ -76,6 +76,7 @@ var (
func init() { func init() {
serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@ -93,11 +94,12 @@ func init() {
filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list") filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.") filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
@ -107,10 +109,12 @@ func init() {
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second") serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files") serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, "<exprimental> enable tcp port")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
@ -137,14 +141,7 @@ func runServer(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false) util.LoadConfiguration("security", false)
util.LoadConfiguration("master", false) util.LoadConfiguration("master", false)
if *serverOptions.cpuprofile != "" { grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile)
f, err := os.Create(*serverOptions.cpuprofile)
if err != nil {
glog.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *isStartingS3 { if *isStartingS3 {
*isStartingFiler = true *isStartingFiler = true
@ -156,19 +153,21 @@ func runServer(cmd *Command, args []string) bool {
*isStartingFiler = true *isStartingFiler = true
} }
_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers) if *isStartingMasterServer {
peers := strings.Join(peerList, ",") _, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
masterOptions.peers = &peers peers := strings.Join(peerList, ",")
masterOptions.peers = &peers
}
// ip address // ip address
masterOptions.ip = serverIp masterOptions.ip = serverIp
masterOptions.ipBind = serverBindIp masterOptions.ipBind = serverBindIp
filerOptions.masters = &peers filerOptions.masters = masterOptions.peers
filerOptions.ip = serverIp filerOptions.ip = serverIp
filerOptions.bindIp = serverBindIp filerOptions.bindIp = serverBindIp
serverOptions.v.ip = serverIp serverOptions.v.ip = serverIp
serverOptions.v.bindIp = serverBindIp serverOptions.v.bindIp = serverBindIp
serverOptions.v.masters = &peers serverOptions.v.masters = masterOptions.peers
serverOptions.v.idleConnectionTimeout = serverTimeout serverOptions.v.idleConnectionTimeout = serverTimeout
serverOptions.v.dataCenter = serverDataCenter serverOptions.v.dataCenter = serverDataCenter
serverOptions.v.rack = serverRack serverOptions.v.rack = serverRack
@ -189,7 +188,6 @@ func runServer(cmd *Command, args []string) bool {
webdavOptions.filer = &filerAddress webdavOptions.filer = &filerAddress
msgBrokerOptions.filer = &filerAddress msgBrokerOptions.filer = &filerAddress
runtime.GOMAXPROCS(runtime.NumCPU())
go stats_collect.StartMetricsServer(*serverMetricsHttpPort) go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
folders := strings.Split(*volumeDataFolders, ",") folders := strings.Split(*volumeDataFolders, ",")

View file

@ -43,7 +43,7 @@ func init() {
upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name") upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit") upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server") upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
} }

View file

@ -6,7 +6,6 @@ import (
"net/http" "net/http"
httppprof "net/http/pprof" httppprof "net/http/pprof"
"os" "os"
"runtime"
"runtime/pprof" "runtime/pprof"
"strconv" "strconv"
"strings" "strings"
@ -36,41 +35,43 @@ var (
) )
type VolumeServerOptions struct { type VolumeServerOptions struct {
port *int port *int
publicPort *int publicPort *int
folders []string folders []string
folderMaxLimits []int folderMaxLimits []int
idxFolder *string idxFolder *string
ip *string ip *string
publicUrl *string publicUrl *string
bindIp *string bindIp *string
masters *string masters *string
idleConnectionTimeout *int idleConnectionTimeout *int
dataCenter *string dataCenter *string
rack *string rack *string
whiteList []string whiteList []string
indexType *string indexType *string
diskType *string diskType *string
fixJpgOrientation *bool fixJpgOrientation *bool
readRedirect *bool readRedirect *bool
cpuProfile *string cpuProfile *string
memProfile *string memProfile *string
compactionMBPerSecond *int compactionMBPerSecond *int
fileSizeLimitMB *int fileSizeLimitMB *int
minFreeSpacePercents []float32 concurrentUploadLimitMB *int
pprof *bool minFreeSpacePercents []float32
preStopSeconds *int pprof *bool
metricsHttpPort *int preStopSeconds *int
metricsHttpPort *int
// pulseSeconds *int // pulseSeconds *int
enableTcp *bool
} }
func init() { func init() {
cmdVolume.Run = runVolume // break init cycle cmdVolume.Run = runVolume // break init cycle
v.port = cmdVolume.Flag.Int("port", 8080, "http listen port") v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public") v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name") v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers") v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") // v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
@ -85,9 +86,11 @@ func init() {
v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second") v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory") v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files") v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<exprimental> enable tcp port")
} }
var cmdVolume = &Command{ var cmdVolume = &Command{
@ -109,8 +112,6 @@ func runVolume(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false) util.LoadConfiguration("security", false)
runtime.GOMAXPROCS(runtime.NumCPU())
// If --pprof is set we assume the caller wants to be able to collect // If --pprof is set we assume the caller wants to be able to collect
// cpu and memory profiles via go tool pprof // cpu and memory profiles via go tool pprof
if !*v.pprof { if !*v.pprof {
@ -238,6 +239,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
*v.fixJpgOrientation, *v.readRedirect, *v.fixJpgOrientation, *v.readRedirect,
*v.compactionMBPerSecond, *v.compactionMBPerSecond,
*v.fileSizeLimitMB, *v.fileSizeLimitMB,
int64(*v.concurrentUploadLimitMB)*1024*1024,
) )
// starting grpc server // starting grpc server
grpcS := v.startGrpcService(volumeServer) grpcS := v.startGrpcService(volumeServer)
@ -251,6 +253,11 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
} }
} }
// starting tcp server
if *v.enableTcp {
go v.startTcpService(volumeServer)
}
// starting the cluster http server // starting the cluster http server
clusterHttpServer := v.startClusterHttpService(volumeMux) clusterHttpServer := v.startClusterHttpService(volumeMux)
@ -368,3 +375,22 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
}() }()
return clusterHttpServer return clusterHttpServer
} }
func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) {
listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port+20000)
glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress)
listener, e := util.NewListener(listeningAddress, 0)
if e != nil {
glog.Fatalf("Volume server listener error on %s:%v", listeningAddress, e)
}
defer listener.Close()
for {
c, err := listener.Accept()
if err != nil {
fmt.Println(err)
return
}
go volumeServer.HandleTcpConnection(c)
}
}

View file

@ -78,7 +78,7 @@ func (wo *WebDavOption) startWebDav() bool {
} }
// parse filer grpc address // parse filer grpc address
filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer) filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*wo.filer)
if err != nil { if err != nil {
glog.Fatal(err) glog.Fatal(err)
return false return false

View file

@ -13,15 +13,15 @@ import (
) )
type SqlGenerator interface { type SqlGenerator interface {
GetSqlInsert(bucket string) string GetSqlInsert(tableName string) string
GetSqlUpdate(bucket string) string GetSqlUpdate(tableName string) string
GetSqlFind(bucket string) string GetSqlFind(tableName string) string
GetSqlDelete(bucket string) string GetSqlDelete(tableName string) string
GetSqlDeleteFolderChildren(bucket string) string GetSqlDeleteFolderChildren(tableName string) string
GetSqlListExclusive(bucket string) string GetSqlListExclusive(tableName string) string
GetSqlListInclusive(bucket string) string GetSqlListInclusive(tableName string) string
GetSqlCreateTable(bucket string) string GetSqlCreateTable(tableName string) string
GetSqlDropTable(bucket string) string GetSqlDropTable(tableName string) string
} }
type AbstractSqlStore struct { type AbstractSqlStore struct {
@ -32,6 +32,29 @@ type AbstractSqlStore struct {
dbsLock sync.Mutex dbsLock sync.Mutex
} }
func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
store.CreateTable(context.Background(), bucket)
if store.dbs == nil {
return
}
store.dbs[bucket] = true
}
func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
store.deleteTable(context.Background(), bucket)
if store.dbs == nil {
return
}
delete(store.dbs, bucket)
}
const ( const (
DEFAULT_TABLE = "filemeta" DEFAULT_TABLE = "filemeta"
) )
@ -253,7 +276,9 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
} }
} }
res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), fullpath) glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath))
if err != nil { if err != nil {
return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err) return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
} }

View file

@ -97,20 +97,20 @@ func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string,
func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) { func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {
var err error var err error
var buffer bytes.Buffer
var shouldRetry bool var shouldRetry bool
receivedData := make([]byte, 0, size)
for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 { for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
for _, urlString := range urlStrings { for _, urlString := range urlStrings {
shouldRetry, err = util.FastReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { receivedData = receivedData[:0]
buffer.Write(data) shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
receivedData = append(receivedData, data...)
}) })
if !shouldRetry { if !shouldRetry {
break break
} }
if err != nil { if err != nil {
glog.V(0).Infof("read %s failed, err: %v", urlString, err) glog.V(0).Infof("read %s failed, err: %v", urlString, err)
buffer.Reset()
} else { } else {
break break
} }
@ -123,7 +123,8 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
} }
} }
return buffer.Bytes(), err return receivedData, err
} }
func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) { func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {

View file

@ -241,12 +241,12 @@ func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err er
if oldEntry != nil { if oldEntry != nil {
entry.Attr.Crtime = oldEntry.Attr.Crtime entry.Attr.Crtime = oldEntry.Attr.Crtime
if oldEntry.IsDirectory() && !entry.IsDirectory() { if oldEntry.IsDirectory() && !entry.IsDirectory() {
glog.Errorf("existing %s is a directory", entry.FullPath) glog.Errorf("existing %s is a directory", oldEntry.FullPath)
return fmt.Errorf("existing %s is a directory", entry.FullPath) return fmt.Errorf("existing %s is a directory", oldEntry.FullPath)
} }
if !oldEntry.IsDirectory() && entry.IsDirectory() { if !oldEntry.IsDirectory() && entry.IsDirectory() {
glog.Errorf("existing %s is a file", entry.FullPath) glog.Errorf("existing %s is a file", oldEntry.FullPath)
return fmt.Errorf("existing %s is a file", entry.FullPath) return fmt.Errorf("existing %s is a file", oldEntry.FullPath)
} }
} }
return f.Store.UpdateEntry(ctx, entry) return f.Store.UpdateEntry(ctx, entry)

View file

@ -29,7 +29,7 @@ func (f *Filer) LoadBuckets() {
limit := int64(math.MaxInt32) limit := int64(math.MaxInt32)
entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "") entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "", "")
if err != nil { if err != nil {
glog.V(1).Infof("no buckets found: %v", err) glog.V(1).Infof("no buckets found: %v", err)

View file

@ -18,6 +18,7 @@ const (
FilerConfName = "filer.conf" FilerConfName = "filer.conf"
IamConfigDirecotry = "/etc/iam" IamConfigDirecotry = "/etc/iam"
IamIdentityFile = "identity.json" IamIdentityFile = "identity.json"
IamPoliciesFile = "policies.json"
) )
type FilerConf struct { type FilerConf struct {

View file

@ -11,6 +11,10 @@ import (
type HardLinkId []byte type HardLinkId []byte
const (
MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
)
func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) { func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) {
if p == "/" { if p == "/" {
return nil return nil
@ -69,7 +73,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
includeLastFile := false includeLastFile := false
if !isDeletingBucket { if !isDeletingBucket {
for { for {
entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "") entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
if err != nil { if err != nil {
glog.Errorf("list folder %s: %v", entry.FullPath, err) glog.Errorf("list folder %s: %v", entry.FullPath, err)
return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)
@ -77,7 +81,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
if lastFileName == "" && !isRecursive && len(entries) > 0 { if lastFileName == "" && !isRecursive && len(entries) > 0 {
// only for first iteration in the loop // only for first iteration in the loop
glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name()) glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
return nil, nil, fmt.Errorf("fail to delete non-empty folder: %s", entry.FullPath) return nil, nil, fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
} }
for _, sub := range entries { for _, sub := range entries {

View file

@ -116,13 +116,13 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
sizeBuf := make([]byte, 4) sizeBuf := make([]byte, 4)
startTsNs := startTime.UnixNano() startTsNs := startTime.UnixNano()
dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "") dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "", "")
if listDayErr != nil { if listDayErr != nil {
return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr) return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
} }
for _, dayEntry := range dayEntries { for _, dayEntry := range dayEntries {
// println("checking day", dayEntry.FullPath) // println("checking day", dayEntry.FullPath)
hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "") hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "", "")
if listHourMinuteErr != nil { if listHourMinuteErr != nil {
return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
} }

View file

@ -11,6 +11,28 @@ import (
// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) { func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
f.maybeReloadFilerConfiguration(event)
f.onBucketEvents(event)
}
func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
message := event.EventNotification
for _, sig := range message.Signatures {
if sig == f.Signature {
return
}
}
if f.DirBucketsPath == event.Directory {
if message.OldEntry == nil && message.NewEntry != nil {
f.Store.OnBucketCreation(message.NewEntry.Name)
}
if message.OldEntry != nil && message.NewEntry == nil {
f.Store.OnBucketDeletion(message.OldEntry.Name)
}
}
}
func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
if DirectoryEtcSeaweedFS != event.Directory { if DirectoryEtcSeaweedFS != event.Directory {
if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath { if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
return return
@ -26,12 +48,11 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
if entry.Name == FilerConfName { if entry.Name == FilerConfName {
f.reloadFilerConfiguration(entry) f.reloadFilerConfiguration(entry)
} }
} }
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -20,9 +20,9 @@ func splitPattern(pattern string) (prefix string, restPattern string) {
} }
// For now, prefix and namePattern are mutually exclusive // For now, prefix and namePattern are mutually exclusive
func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string) (entries []*Entry, hasMore bool, err error) { func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) {
_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, func(entry *Entry) bool { _, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
entries = append(entries, entry) entries = append(entries, entry)
return true return true
}) })
@ -36,7 +36,7 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
} }
// For now, prefix and namePattern are mutually exclusive // For now, prefix and namePattern are mutually exclusive
func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) { func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
if strings.HasSuffix(string(p), "/") && len(p) > 1 { if strings.HasSuffix(string(p), "/") && len(p) > 1 {
p = p[0 : len(p)-1] p = p[0 : len(p)-1]
} }
@ -47,30 +47,38 @@ func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath,
} }
var missedCount int64 var missedCount int64
missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, eachEntryFunc) missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
for missedCount > 0 && err == nil { for missedCount > 0 && err == nil {
missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, eachEntryFunc) missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
} }
return return
} }
func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) { func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
if len(restNamePattern) == 0 { if len(restNamePattern) == 0 && len(namePatternExclude) == 0{
lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc) lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
return 0, lastFileName, err return 0, lastFileName, err
} }
lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool { lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) bool {
nameToTest := strings.ToLower(entry.Name()) nameToTest := entry.Name()
if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && matched { if len(namePatternExclude) > 0 {
if !eachEntryFunc(entry) { if matched, matchErr := filepath.Match(namePatternExclude, nameToTest); matchErr == nil && matched {
return false missedCount++
return true
} }
} else { }
missedCount++ if len(restNamePattern) > 0 {
if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && !matched {
missedCount++
return true
}
}
if !eachEntryFunc(entry) {
return false
} }
return true return true
}) })

View file

@ -39,3 +39,8 @@ type FilerStore interface {
Shutdown() Shutdown()
} }
type BucketAware interface {
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
}

View file

@ -21,6 +21,8 @@ type VirtualFilerStore interface {
DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
DeleteOneEntry(ctx context.Context, entry *Entry) error DeleteOneEntry(ctx context.Context, entry *Entry) error
AddPathSpecificStore(path string, storeId string, store FilerStore) AddPathSpecificStore(path string, storeId string, store FilerStore)
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
} }
type FilerStoreWrapper struct { type FilerStoreWrapper struct {
@ -40,6 +42,27 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
} }
} }
func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
for _, store := range fsw.storeIdToStore {
if ba, ok := store.(BucketAware); ok {
ba.OnBucketCreation(bucket)
}
}
if ba, ok := fsw.defaultStore.(BucketAware); ok {
ba.OnBucketCreation(bucket)
}
}
func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) {
for _, store := range fsw.storeIdToStore {
if ba, ok := store.(BucketAware); ok {
ba.OnBucketDeletion(bucket)
}
}
if ba, ok := fsw.defaultStore.(BucketAware); ok {
ba.OnBucketDeletion(bucket)
}
}
func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) { func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store) fsw.storeIdToStore[storeId] = NewFilerStorePathTranlator(path, store)
err := fsw.pathToStore.Put([]byte(path), storeId) err := fsw.pathToStore.Put([]byte(path), storeId)
@ -126,8 +149,8 @@ func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (
stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "find").Observe(time.Since(start).Seconds()) stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "find").Observe(time.Since(start).Seconds())
}() }()
glog.V(4).Infof("FindEntry %s", fp)
entry, err = actualStore.FindEntry(ctx, fp) entry, err = actualStore.FindEntry(ctx, fp)
glog.V(4).Infof("FindEntry %s: %v", fp, err)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -8,6 +8,7 @@ import (
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util" leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"os"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
@ -38,6 +39,7 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, pre
func (store *LevelDBStore) initialize(dir string) (err error) { func (store *LevelDBStore) initialize(dir string) (err error) {
glog.Infof("filer store dir: %s", dir) glog.Infof("filer store dir: %s", dir)
os.MkdirAll(dir, 0755)
if err := weed_util.TestFolderWritable(dir); err != nil { if err := weed_util.TestFolderWritable(dir); err != nil {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
} }

View file

@ -51,14 +51,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -77,7 +77,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

View file

@ -38,6 +38,7 @@ func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, pr
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
glog.Infof("filer store leveldb2 dir: %s", dir) glog.Infof("filer store leveldb2 dir: %s", dir)
os.MkdirAll(dir, 0755)
if err := weed_util.TestFolderWritable(dir); err != nil { if err := weed_util.TestFolderWritable(dir); err != nil {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
} }

View file

@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

View file

@ -45,6 +45,7 @@ func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, pr
func (store *LevelDB3Store) initialize(dir string) (err error) { func (store *LevelDB3Store) initialize(dir string) (err error) {
glog.Infof("filer store leveldb3 dir: %s", dir) glog.Infof("filer store leveldb3 dir: %s", dir)
os.MkdirAll(dir, 0755)
if err := weed_util.TestFolderWritable(dir); err != nil { if err := weed_util.TestFolderWritable(dir); err != nil {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
} }

View file

@ -49,14 +49,14 @@ func TestCreateAndFind(t *testing.T) {
} }
// checking one upper directory // checking one upper directory
entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "") entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
} }
// checking one upper directory // checking one upper directory
entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if len(entries) != 1 { if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries)) t.Errorf("list entries count: %v", len(entries))
return return
@ -75,7 +75,7 @@ func TestEmptyRoot(t *testing.T) {
ctx := context.Background() ctx := context.Background()
// checking one upper directory // checking one upper directory
entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "") entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
if err != nil { if err != nil {
t.Errorf("list entries: %v", err) t.Errorf("list entries: %v", err)
return return

View file

@ -69,6 +69,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
peerSignature, err = ma.readFilerStoreSignature(peer) peerSignature, err = ma.readFilerStoreSignature(peer)
} }
// when filer store is not shared by multiple filers
if peerSignature != f.Signature { if peerSignature != f.Signature {
if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil { if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
lastTsNs = prevTsNs lastTsNs = prevTsNs

View file

@ -2,6 +2,7 @@ package mysql
import ( import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
) )
@ -9,44 +10,49 @@ import (
type SqlGenMysql struct { type SqlGenMysql struct {
CreateTableSqlTemplate string CreateTableSqlTemplate string
DropTableSqlTemplate string DropTableSqlTemplate string
UpsertQueryTemplate string
} }
var ( var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{}) _ = abstract_sql.SqlGenerator(&SqlGenMysql{})
) )
func (gen *SqlGenMysql) GetSqlInsert(bucket string) string { func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", bucket) if gen.UpsertQueryTemplate != "" {
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
} else {
return fmt.Sprintf("INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?)", tableName)
}
} }
func (gen *SqlGenMysql) GetSqlUpdate(bucket string) string { func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", bucket) return fmt.Sprintf("UPDATE `%s` SET meta=? WHERE dirhash=? AND name=? AND directory=?", tableName)
} }
func (gen *SqlGenMysql) GetSqlFind(bucket string) string { func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket) return fmt.Sprintf("SELECT meta FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
} }
func (gen *SqlGenMysql) GetSqlDelete(bucket string) string { func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", bucket) return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND name=? AND directory=?", tableName)
} }
func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(bucket string) string { func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", bucket) return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
} }
func (gen *SqlGenMysql) GetSqlListExclusive(bucket string) string { func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket) return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
} }
func (gen *SqlGenMysql) GetSqlListInclusive(bucket string) string { func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", bucket) return fmt.Sprintf("SELECT NAME, meta FROM `%s` WHERE dirhash=? AND name>=? AND directory=? AND name like ? ORDER BY NAME ASC LIMIT ?", tableName)
} }
func (gen *SqlGenMysql) GetSqlCreateTable(bucket string) string { func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket) return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
} }
func (gen *SqlGenMysql) GetSqlDropTable(bucket string) string { func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
return fmt.Sprintf(gen.DropTableSqlTemplate, bucket) return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
} }

View file

@ -3,9 +3,10 @@ package mysql
import ( import (
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql" "github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
@ -29,6 +30,8 @@ func (store *MysqlStore) GetName() string {
func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -41,13 +44,17 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str
) )
} }
func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen, func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) { maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = false store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
}
store.SqlGenerator = &SqlGenMysql{ store.SqlGenerator = &SqlGenMysql{
CreateTableSqlTemplate: "", CreateTableSqlTemplate: "",
DropTableSqlTemplate: "drop table `%s`", DropTableSqlTemplate: "drop table `%s`",
UpsertQueryTemplate: upsertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

View file

@ -32,6 +32,8 @@ func (store *MysqlStore2) GetName() string {
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"createTable"), configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -44,13 +46,17 @@ func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix st
) )
} }
func (store *MysqlStore2) initialize(createTable, user, password, hostname string, port int, database string, maxIdle, maxOpen, func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
maxLifetimeSeconds int, interpolateParams bool) (err error) { maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
}
store.SqlGenerator = &mysql.SqlGenMysql{ store.SqlGenerator = &mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable, CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table `%s`", DropTableSqlTemplate: "drop table `%s`",
UpsertQueryTemplate: upsertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)

View file

@ -10,44 +10,49 @@ import (
type SqlGenPostgres struct { type SqlGenPostgres struct {
CreateTableSqlTemplate string CreateTableSqlTemplate string
DropTableSqlTemplate string DropTableSqlTemplate string
UpsertQueryTemplate string
} }
var ( var (
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{}) _ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
) )
func (gen *SqlGenPostgres) GetSqlInsert(bucket string) string { func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, bucket) if gen.UpsertQueryTemplate != "" {
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
} else {
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
}
} }
func (gen *SqlGenPostgres) GetSqlUpdate(bucket string) string { func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, bucket) return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlFind(bucket string) string { func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket) return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlDelete(bucket string) string { func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, bucket) return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(bucket string) string { func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, bucket) return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlListExclusive(bucket string) string { func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket) return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlListInclusive(bucket string) string { func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, bucket) return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND name>=$2 AND directory=$3 AND name like $4 ORDER BY NAME ASC LIMIT $5`, tableName)
} }
func (gen *SqlGenPostgres) GetSqlCreateTable(bucket string) string { func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
return fmt.Sprintf(gen.CreateTableSqlTemplate, bucket) return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
} }
func (gen *SqlGenPostgres) GetSqlDropTable(bucket string) string { func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
return fmt.Sprintf(gen.DropTableSqlTemplate, bucket) return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
} }

View file

@ -29,6 +29,8 @@ func (store *PostgresStore) GetName() string {
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -42,12 +44,16 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
) )
} }
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
}
store.SqlGenerator = &SqlGenPostgres{ store.SqlGenerator = &SqlGenPostgres{
CreateTableSqlTemplate: "", CreateTableSqlTemplate: "",
DropTableSqlTemplate: `drop table "%s"`, DropTableSqlTemplate: `drop table "%s"`,
UpsertQueryTemplate: upsertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

View file

@ -32,6 +32,8 @@ func (store *PostgresStore2) GetName() string {
func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) { func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
return store.initialize( return store.initialize(
configuration.GetString(prefix+"createTable"), configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"), configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"), configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"), configuration.GetString(prefix+"hostname"),
@ -45,12 +47,16 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
) )
} }
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true store.SupportBucketTable = true
if !enableUpsert {
upsertQuery = ""
}
store.SqlGenerator = &postgres.SqlGenPostgres{ store.SqlGenerator = &postgres.SqlGenPostgres{
CreateTableSqlTemplate: createTable, CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: `drop table "%s"`, DropTableSqlTemplate: `drop table "%s"`,
UpsertQueryTemplate: upsertQuery,
} }
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode) sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, sslmode)

Some files were not shown because too many files have changed in this diff Show more