gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into mq-subscribe

Browse source
This commit is contained in:
chrislu 2024-01-11 20:58:57 -08:00
parent 188f471a24 0e8a54f6f6
commit 70a011a224
37 changed files with 175 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/binaries_dev.yml vendored
View file

@ -44,7 +44,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk - name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -60,7 +60,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -93,7 +93,7 @@ jobs:
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk - name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -109,7 +109,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

4
.github/workflows/binaries_release0.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -44,7 +44,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

4
.github/workflows/binaries_release1.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -44,7 +44,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

4
.github/workflows/binaries_release2.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -44,7 +44,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

4
.github/workflows/binaries_release3.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -44,7 +44,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

4
.github/workflows/binaries_release4.yml vendored
View file

@ -30,7 +30,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2 - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -45,7 +45,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22 uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

2
.github/workflows/container_dev.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs

2
.github/workflows/container_latest.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs

2
.github/workflows/container_release1.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs

2
.github/workflows/container_release2.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs

2
.github/workflows/container_release3.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs

4
.github/workflows/container_release4.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -52,7 +52,7 @@ jobs:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build file: ./docker/Dockerfile.go_build
build-args: TAGS=elastic,ydb,gocdk,tikv build-args: TAGS=elastic,ydb,gocdk,tikv,rclone
platforms: linux/amd64 platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }} tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }} labels: ${{ steps.docker_meta.outputs.labels }}

4
.github/workflows/container_release5.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -52,7 +52,7 @@ jobs:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build file: ./docker/Dockerfile.go_build
build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv,rclone
platforms: linux/amd64 platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }} tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }} labels: ${{ steps.docker_meta.outputs.labels }}

2
.github/workflows/depsreview.yml vendored
View file

@ -11,4 +11,4 @@ jobs:
- name: 'Checkout Repository' - name: 'Checkout Repository'
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: 'Dependency Review' - name: 'Dependency Review'
uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4 uses: actions/dependency-review-action@c74b580d73376b7750d3d2a50bfb8adc2c937507

2
Makefile
View file

@ -15,7 +15,7 @@ full_install:
cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone" cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
server: install server: install
weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324
benchmark: install warp_install benchmark: install warp_install
pkill weed || true pkill weed || true

26
go.mod
View file

@ -111,7 +111,7 @@ require (
gocloud.dev/pubsub/rabbitpubsub v0.36.0 gocloud.dev/pubsub/rabbitpubsub v0.36.0
golang.org/x/crypto v0.17.0 // indirect golang.org/x/crypto v0.17.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/image v0.14.0 golang.org/x/image v0.15.0
golang.org/x/net v0.19.0 golang.org/x/net v0.19.0
golang.org/x/oauth2 v0.14.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect
golang.org/x/sys v0.15.0 golang.org/x/sys v0.15.0
@ -121,7 +121,7 @@ require (
google.golang.org/api v0.151.0 google.golang.org/api v0.151.0
google.golang.org/appengine v1.6.8 // indirect google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/grpc v1.59.0 google.golang.org/grpc v1.60.1
google.golang.org/protobuf v1.32.0 google.golang.org/protobuf v1.32.0
gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect modernc.org/b v1.0.0 // indirect
@ -140,9 +140,9 @@ require (
github.com/Jille/raft-grpc-transport v1.5.0 github.com/Jille/raft-grpc-transport v1.5.0
github.com/arangodb/go-driver v1.6.1 github.com/arangodb/go-driver v1.6.1
github.com/armon/go-metrics v0.4.1 github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2 v1.24.1
github.com/aws/aws-sdk-go-v2/config v1.26.2 github.com/aws/aws-sdk-go-v2/config v1.26.2
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 github.com/aws/aws-sdk-go-v2/credentials v1.16.14
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5
github.com/fluent/fluent-logger-golang v1.9.0 github.com/fluent/fluent-logger-golang v1.9.0
github.com/golang-jwt/jwt/v5 v5.1.0 github.com/golang-jwt/jwt/v5 v5.1.0
@ -188,26 +188,26 @@ require (
github.com/andybalholm/cascadia v1.3.2 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
github.com/aws/smithy-go v1.19.0 // indirect github.com/aws/smithy-go v1.19.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.13.1 // indirect github.com/bradenaw/juniper v0.13.1 // indirect
github.com/buengese/sgzip v0.1.1 // indirect github.com/buengese/sgzip v0.1.1 // indirect
github.com/calebcase/tmpfile v1.0.3 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect
github.com/cloudflare/circl v1.3.6 // indirect github.com/cloudflare/circl v1.3.7 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 // indirect github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect
@ -226,7 +226,7 @@ require (
github.com/geoffgarside/ber v1.1.0 // indirect github.com/geoffgarside/ber v1.1.0 // indirect
github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-resty/resty/v2 v2.10.0 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect

52
go.sum
View file

@ -141,20 +141,20 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY= github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY=
github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc= github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8= github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8= github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE=
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw= github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
@ -163,8 +163,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY=
@ -173,12 +173,12 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 h1:umyC9zH/A1w8AXrrG7iMxT4Rfgj8
github.com/aws/aws-sdk-go-v2/service/sns v1.26.5/go.mod h1:IrcbquqMupzndZ20BXxDxjM7XenTRhbwBOetk4+Z5oc= github.com/aws/aws-sdk-go-v2/service/sns v1.26.5/go.mod h1:IrcbquqMupzndZ20BXxDxjM7XenTRhbwBOetk4+Z5oc=
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 h1:cJb4I498c1mrOVrRqYTcnLD65AFqUuseHfzHdNZHL9U= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 h1:cJb4I498c1mrOVrRqYTcnLD65AFqUuseHfzHdNZHL9U=
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5/go.mod h1:mCUv04gd/7g+/HNzDB4X6dzJuygji0ckvB3Lg/TdG5Y= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5/go.mod h1:mCUv04gd/7g+/HNzDB4X6dzJuygji0ckvB3Lg/TdG5Y=
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac=
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@ -221,8 +221,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 h1:KzZU0EMkUm4vX/jPp5d/VttocDpocL/8QP0zyiI9Xiw= github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 h1:KzZU0EMkUm4vX/jPp5d/VttocDpocL/8QP0zyiI9Xiw=
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q= github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@ -361,8 +361,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4+R14N6k= github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4+R14N6k=
github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc=
github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@ -1073,8 +1073,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1467,8 +1467,8 @@ google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk=
google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE= google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE=

4
weed/filer/stream.go
View file

@ -121,10 +121,10 @@ func StreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, w
remaining -= int64(chunkView.ViewSize) remaining -= int64(chunkView.ViewSize)
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
if err != nil { if err != nil {
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc() stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
return fmt.Errorf("read chunk: %v", err) return fmt.Errorf("read chunk: %v", err)
} }
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc() stats.FilerHandlerCounter.WithLabelValues("chunkDownload").Inc()
downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize)) downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize))
} }
if remaining > 0 { if remaining > 0 {

2
weed/operation/upload_content.go
View file

@ -329,7 +329,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
if strings.Contains(post_err.Error(), "connection reset by peer") || if strings.Contains(post_err.Error(), "connection reset by peer") ||
strings.Contains(post_err.Error(), "use of closed network connection") { strings.Contains(post_err.Error(), "use of closed network connection") {
glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr) glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr)
stats.FilerRequestCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc()
resp, post_err = HttpClient.Do(req) resp, post_err = HttpClient.Do(req)
defer util.CloseResponse(resp) defer util.CloseResponse(resp)
} }

20
weed/s3api/stats.go
View file

@ -8,29 +8,11 @@ import (
"time" "time"
) )
type StatusRecorder struct {
http.ResponseWriter
Status int
}
func NewStatusResponseWriter(w http.ResponseWriter) *StatusRecorder {
return &StatusRecorder{w, http.StatusOK}
}
func (r *StatusRecorder) WriteHeader(status int) {
r.Status = status
r.ResponseWriter.WriteHeader(status)
}
func (r *StatusRecorder) Flush() {
r.ResponseWriter.(http.Flusher).Flush()
}
func track(f http.HandlerFunc, action string) http.HandlerFunc { func track(f http.HandlerFunc, action string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r) bucket, _ := s3_constants.GetBucketAndObject(r)
w.Header().Set("Server", "SeaweedFS S3") w.Header().Set("Server", "SeaweedFS S3")
recorder := NewStatusResponseWriter(w) recorder := stats_collect.NewStatusResponseWriter(w)
start := time.Now() start := time.Now()
f(recorder, r) f(recorder, r)
if recorder.Status == http.StatusForbidden { if recorder.Status == http.StatusForbidden {

8
weed/server/filer_server.go
View file

@ -111,11 +111,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
v.SetDefault("cors.allowed_origins.values", "*") v.SetDefault("cors.allowed_origins.values", "*")
if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) { allowedOrigins := v.GetString("cors.allowed_origins.values")
allowedOrigins := v.GetString("cors.allowed_origins.values") domains := strings.Split(allowedOrigins, ",")
domains := strings.Split(allowedOrigins, ",") option.AllowedOrigins = domains
option.AllowedOrigins = domains
}
fs = &FilerServer{ fs = &FilerServer{
option: option, option: option,

20
weed/server/filer_server_handlers.go
View file

@ -4,6 +4,7 @@ import (
"errors" "errors"
"net/http" "net/http"
"os" "os"
"strconv"
"strings" "strings"
"sync/atomic" "sync/atomic"
"time" "time"
@ -17,7 +18,8 @@ import (
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
start := time.Now() start := time.Now()
statusRecorder := stats.NewStatusResponseWriter(w)
w = statusRecorder
origin := r.Header.Get("Origin") origin := r.Header.Get("Origin")
if origin != "" { if origin != "" {
if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" { if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" {
@ -53,23 +55,23 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
fileId = r.RequestURI[len("/?proxyChunkId="):] fileId = r.RequestURI[len("/?proxyChunkId="):]
} }
if fileId != "" { if fileId != "" {
stats.FilerRequestCounter.WithLabelValues(stats.ChunkProxy).Inc()
fs.proxyToVolumeServer(w, r, fileId) fs.proxyToVolumeServer(w, r, fileId)
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkProxy).Inc()
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkProxy).Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues(stats.ChunkProxy).Observe(time.Since(start).Seconds())
return return
} }
defer func() {
stats.FilerRequestCounter.WithLabelValues(r.Method, strconv.Itoa(statusRecorder.Status)).Inc()
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
}()
isReadHttpCall := r.Method == "GET" || r.Method == "HEAD" isReadHttpCall := r.Method == "GET" || r.Method == "HEAD"
if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) { if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) {
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
return return
} }
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
defer func() {
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
}()
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
switch r.Method { switch r.Method {
@ -115,6 +117,8 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
start := time.Now() start := time.Now()
statusRecorder := stats.NewStatusResponseWriter(w)
w = statusRecorder
os.Stdout.WriteString("Request: " + r.Method + " " + r.URL.String() + "\n") os.Stdout.WriteString("Request: " + r.Method + " " + r.URL.String() + "\n")
@ -140,8 +144,8 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
w.Header().Set("Access-Control-Allow-Credentials", "true") w.Header().Set("Access-Control-Allow-Credentials", "true")
} }
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
defer func() { defer func() {
stats.FilerRequestCounter.WithLabelValues(r.Method, strconv.Itoa(statusRecorder.Status)).Inc()
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
}() }()
// We handle OPTIONS first because it never should be authenticated // We handle OPTIONS first because it never should be authenticated

10
weed/server/filer_server_handlers_read.go
View file

@ -98,11 +98,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
} }
if err == filer_pb.ErrNotFound { if err == filer_pb.ErrNotFound {
glog.V(2).Infof("Not found %s: %v", path, err) glog.V(2).Infof("Not found %s: %v", path, err)
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadNotFound).Inc()
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
} else { } else {
glog.Errorf("Internal %s: %v", path, err) glog.Errorf("Internal %s: %v", path, err)
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadInternal).Inc()
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
} }
return return
@ -233,7 +233,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if offset+size <= int64(len(entry.Content)) { if offset+size <= int64(len(entry.Content)) {
_, err := writer.Write(entry.Content[offset : offset+size]) _, err := writer.Write(entry.Content[offset : offset+size])
if err != nil { if err != nil {
stats.FilerRequestCounter.WithLabelValues(stats.ErrorWriteEntry).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ErrorWriteEntry).Inc()
glog.Errorf("failed to write entry content: %v", err) glog.Errorf("failed to write entry content: %v", err)
} }
return err return err
@ -245,7 +245,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
Directory: dir, Directory: dir,
Name: name, Name: name,
}); err != nil { }); err != nil {
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadCache).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadCache).Inc()
glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err) glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
return fmt.Errorf("cache %s: %v", entry.FullPath, err) return fmt.Errorf("cache %s: %v", entry.FullPath, err)
} else { } else {
@ -255,7 +255,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
err = filer.StreamContentWithThrottler(fs.filer.MasterClient, writer, chunks, offset, size, fs.option.DownloadMaxBytesPs) err = filer.StreamContentWithThrottler(fs.filer.MasterClient, writer, chunks, offset, size, fs.option.DownloadMaxBytesPs)
if err != nil { if err != nil {
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadStream).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadStream).Inc()
glog.Errorf("failed to stream content %s: %v", r.URL, err) glog.Errorf("failed to stream content %s: %v", r.URL, err)
} }
return err return err

2
weed/server/filer_server_handlers_read_dir.go
View file

@ -18,7 +18,7 @@ import (
// is empty. // is empty.
func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) {
stats.FilerRequestCounter.WithLabelValues(stats.DirList).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.DirList).Inc()
path := r.URL.Path path := r.URL.Path
if strings.HasSuffix(path, "/") && len(path) > 1 { if strings.HasSuffix(path, "/") && len(path) > 1 {

2
weed/server/filer_server_handlers_write.go
View file

@ -35,7 +35,7 @@ type FilerPostResult struct {
func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) { func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssign).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssign).Inc()
start := time.Now() start := time.Now()
defer func() { defer func() {
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkAssign).Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues(stats.ChunkAssign).Observe(time.Since(start).Seconds())

12
weed/server/filer_server_handlers_write_upload.go
View file

@ -86,11 +86,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
bytesBuffer.Read(smallContent) bytesBuffer.Read(smallContent)
bufPool.Put(bytesBuffer) bufPool.Put(bytesBuffer)
<-bytesBufferLimitChan <-bytesBufferLimitChan
stats.FilerRequestCounter.WithLabelValues(stats.ContentSaveToFiler).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ContentSaveToFiler).Inc()
break break
} }
} else { } else {
stats.FilerRequestCounter.WithLabelValues(stats.AutoChunk).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.AutoChunk).Inc()
} }
wg.Add(1) wg.Add(1)
@ -143,7 +143,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) { func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
stats.FilerRequestCounter.WithLabelValues(stats.ChunkUpload).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUpload).Inc()
start := time.Now() start := time.Now()
defer func() { defer func() {
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkUpload).Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues(stats.ChunkUpload).Observe(time.Since(start).Seconds())
@ -160,7 +160,7 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
} }
uploadResult, err, data := operation.Upload(limitedReader, uploadOption) uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
if uploadResult != nil && uploadResult.RetryCount > 0 { if uploadResult != nil && uploadResult.RetryCount > 0 {
stats.FilerRequestCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount)) stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount))
} }
return uploadResult, err, data return uploadResult, err, data
} }
@ -180,14 +180,14 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so) fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so)
if uploadErr != nil { if uploadErr != nil {
glog.V(4).Infof("retry later due to assign error: %v", uploadErr) glog.V(4).Infof("retry later due to assign error: %v", uploadErr)
stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssignRetry).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
return uploadErr return uploadErr
} }
// upload the chunk to the volume server // upload the chunk to the volume server
uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth) uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth)
if uploadErr != nil { if uploadErr != nil {
glog.V(4).Infof("retry later due to upload error: %v", uploadErr) glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc() stats.FilerHandlerCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()
fid, _ := filer_pb.ToFileIdObject(fileId) fid, _ := filer_pb.ToFileIdObject(fileId)
fileChunk := filer_pb.FileChunk{ fileChunk := filer_pb.FileChunk{
FileId: fileId, FileId: fileId,

4
weed/server/master_grpc_server.go
View file

@ -168,10 +168,10 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
DataCenter: dn.GetDataCenterId(), DataCenter: dn.GetDataCenterId(),
} }
if len(heartbeat.NewVolumes) > 0 { if len(heartbeat.NewVolumes) > 0 {
stats.FilerRequestCounter.WithLabelValues("newVolumes").Inc() stats.MasterReceivedHeartbeatCounter.WithLabelValues("newVolumes").Inc()
} }
if len(heartbeat.DeletedVolumes) > 0 { if len(heartbeat.DeletedVolumes) > 0 {
stats.FilerRequestCounter.WithLabelValues("deletedVolumes").Inc() stats.MasterReceivedHeartbeatCounter.WithLabelValues("deletedVolumes").Inc()
} }
if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 { if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
// process delta volume ids if exists for fast volume id updates // process delta volume ids if exists for fast volume id updates

20
weed/server/volume_server_handlers.go
View file

@ -31,6 +31,8 @@ security settings:
*/ */
func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Request) {
statusRecorder := stats.NewStatusResponseWriter(w)
w = statusRecorder
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
if r.Header.Get("Origin") != "" { if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Origin", "*")
@ -38,10 +40,10 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
} }
start := time.Now() start := time.Now()
requestMethod := r.Method requestMethod := r.Method
defer func(start time.Time, method *string) { defer func(start time.Time, method *string, statusRecorder *stats.StatusRecorder) {
stats.VolumeServerRequestCounter.WithLabelValues(*method).Inc() stats.VolumeServerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds()) stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
}(start, &requestMethod) }(start, &requestMethod, statusRecorder)
switch r.Method { switch r.Method {
case http.MethodGet, http.MethodHead: case http.MethodGet, http.MethodHead:
stats.ReadRequest() stats.ReadRequest()
@ -63,11 +65,9 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
vs.inFlightDownloadDataLimitCond.L.Unlock() vs.inFlightDownloadDataLimitCond.L.Unlock()
vs.GetOrHeadHandler(w, r) vs.GetOrHeadHandler(w, r)
case http.MethodDelete: case http.MethodDelete:
stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
stats.DeleteRequest() stats.DeleteRequest()
vs.guard.WhiteList(vs.DeleteHandler)(w, r) vs.guard.WhiteList(vs.DeleteHandler)(w, r)
case http.MethodPut, http.MethodPost: case http.MethodPut, http.MethodPost:
stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
contentLength := getContentLength(r) contentLength := getContentLength(r)
// exclude the replication from the concurrentUploadLimitMB // exclude the replication from the concurrentUploadLimitMB
if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 { if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
@ -124,11 +124,21 @@ func getContentLength(r *http.Request) int64 {
} }
func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) { func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
statusRecorder := stats.NewStatusResponseWriter(w)
w = statusRecorder
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
if r.Header.Get("Origin") != "" { if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true") w.Header().Set("Access-Control-Allow-Credentials", "true")
} }
start := time.Now()
requestMethod := r.Method
defer func(start time.Time, method *string, statusRecorder *stats.StatusRecorder) {
stats.VolumeServerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
}(start, &requestMethod, statusRecorder)
switch r.Method { switch r.Method {
case http.MethodGet, http.MethodHead: case http.MethodGet, http.MethodHead:
stats.ReadRequest() stats.ReadRequest()

4
weed/server/volume_server_handlers_read.go
View file

@ -30,12 +30,12 @@ import (
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
func NotFound(w http.ResponseWriter) { func NotFound(w http.ResponseWriter) {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorGetNotFound).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetNotFound).Inc()
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
} }
func InternalError(w http.ResponseWriter) { func InternalError(w http.ResponseWriter) {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorGetInternal).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetInternal).Inc()
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
} }

21
weed/stats/http_status_recorder.go Normal file
View file

@ -0,0 +1,21 @@
package stats
import "net/http"
type StatusRecorder struct {
http.ResponseWriter
Status int
}
func NewStatusResponseWriter(w http.ResponseWriter) *StatusRecorder {
return &StatusRecorder{w, http.StatusOK}
}
func (r *StatusRecorder) WriteHeader(status int) {
r.Status = status
r.ResponseWriter.WriteHeader(status)
}
func (r *StatusRecorder) Flush() {
r.ResponseWriter.(http.Flusher).Flush()
}

18
weed/stats/metrics.go
View file

@ -84,6 +84,14 @@ var (
Subsystem: "filer", Subsystem: "filer",
Name: "request_total", Name: "request_total",
Help: "Counter of filer requests.", Help: "Counter of filer requests.",
}, []string{"type", "code"})
FilerHandlerCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: "filer",
Name: "handler_total",
Help: "Counter of filer handlers.",
}, []string{"type"}) }, []string{"type"})
FilerRequestHistogram = prometheus.NewHistogramVec( FilerRequestHistogram = prometheus.NewHistogramVec(
@ -134,6 +142,14 @@ var (
Subsystem: "volumeServer", Subsystem: "volumeServer",
Name: "request_total", Name: "request_total",
Help: "Counter of volume server requests.", Help: "Counter of volume server requests.",
}, []string{"type", "code"})
VolumeServerHandlerCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: "volumeServer",
Name: "handler_total",
Help: "Counter of volume server handlers.",
}, []string{"type"}) }, []string{"type"})
VolumeServerVacuumingCompactCounter = prometheus.NewCounterVec( VolumeServerVacuumingCompactCounter = prometheus.NewCounterVec(
@ -245,6 +261,7 @@ func init() {
Gather.MustRegister(MasterReplicaPlacementMismatch) Gather.MustRegister(MasterReplicaPlacementMismatch)
Gather.MustRegister(FilerRequestCounter) Gather.MustRegister(FilerRequestCounter)
Gather.MustRegister(FilerHandlerCounter)
Gather.MustRegister(FilerRequestHistogram) Gather.MustRegister(FilerRequestHistogram)
Gather.MustRegister(FilerStoreCounter) Gather.MustRegister(FilerStoreCounter)
Gather.MustRegister(FilerStoreHistogram) Gather.MustRegister(FilerStoreHistogram)
@ -254,6 +271,7 @@ func init() {
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
Gather.MustRegister(VolumeServerRequestCounter) Gather.MustRegister(VolumeServerRequestCounter)
Gather.MustRegister(VolumeServerHandlerCounter)
Gather.MustRegister(VolumeServerRequestHistogram) Gather.MustRegister(VolumeServerRequestHistogram)
Gather.MustRegister(VolumeServerVacuumingCompactCounter) Gather.MustRegister(VolumeServerVacuumingCompactCounter)
Gather.MustRegister(VolumeServerVacuumingCommitCounter) Gather.MustRegister(VolumeServerVacuumingCommitCounter)

8
weed/storage/disk_location_ec.go
View file

@ -218,9 +218,13 @@ func (l *DiskLocation) unmountEcVolumeByCollection(collectionName string) map[ne
return deltaVols return deltaVols
} }
func (l *DiskLocation) EcVolumesLen() int { func (l *DiskLocation) EcShardCount() int {
l.ecVolumesLock.RLock() l.ecVolumesLock.RLock()
defer l.ecVolumesLock.RUnlock() defer l.ecVolumesLock.RUnlock()
return len(l.ecVolumes) shardCount := 0
for _, ecVolume := range l.ecVolumes {
shardCount += len(ecVolume.Shards)
}
return shardCount
} }

20
weed/storage/needle/needle_read.go
View file

@ -54,11 +54,11 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
if n.Size != size { if n.Size != size {
// cookie is not always passed in for this API. Use size to do preliminary checking. // cookie is not always passed in for this API. Use size to do preliminary checking.
if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc()
glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
return ErrorSizeMismatch return ErrorSizeMismatch
} }
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc()
return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
} }
switch version { switch version {
@ -75,7 +75,7 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
newChecksum := NewCRC(n.Data) newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() && checksum != uint32(newChecksum) { if checksum != newChecksum.Value() && checksum != uint32(newChecksum) {
// the crc.Value() function is to be deprecated. this double checking is for backward compatible. // the crc.Value() function is to be deprecated. this double checking is for backward compatible.
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorCRC).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc()
return errors.New("CRC error! Data On Disk Corrupted") return errors.New("CRC error! Data On Disk Corrupted")
} }
n.Checksum = newChecksum n.Checksum = newChecksum
@ -108,7 +108,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
n.DataSize = util.BytesToUint32(bytes[index : index+4]) n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4 index = index + 4
if int(n.DataSize)+index > lenBytes { if int(n.DataSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return fmt.Errorf("index out of range %d", 1) return fmt.Errorf("index out of range %d", 1)
} }
n.Data = bytes[index : index+int(n.DataSize)] n.Data = bytes[index : index+int(n.DataSize)]
@ -127,7 +127,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
n.NameSize = uint8(bytes[index]) n.NameSize = uint8(bytes[index])
index = index + 1 index = index + 1
if int(n.NameSize)+index > lenBytes { if int(n.NameSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 2) return index, fmt.Errorf("index out of range %d", 2)
} }
n.Name = bytes[index : index+int(n.NameSize)] n.Name = bytes[index : index+int(n.NameSize)]
@ -137,7 +137,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
n.MimeSize = uint8(bytes[index]) n.MimeSize = uint8(bytes[index])
index = index + 1 index = index + 1
if int(n.MimeSize)+index > lenBytes { if int(n.MimeSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 3) return index, fmt.Errorf("index out of range %d", 3)
} }
n.Mime = bytes[index : index+int(n.MimeSize)] n.Mime = bytes[index : index+int(n.MimeSize)]
@ -145,7 +145,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
} }
if index < lenBytes && n.HasLastModifiedDate() { if index < lenBytes && n.HasLastModifiedDate() {
if LastModifiedBytesLength+index > lenBytes { if LastModifiedBytesLength+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 4) return index, fmt.Errorf("index out of range %d", 4)
} }
n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
@ -153,7 +153,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
} }
if index < lenBytes && n.HasTtl() { if index < lenBytes && n.HasTtl() {
if TtlBytesLength+index > lenBytes { if TtlBytesLength+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 5) return index, fmt.Errorf("index out of range %d", 5)
} }
n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength]) n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
@ -161,13 +161,13 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
} }
if index < lenBytes && n.HasPairs() { if index < lenBytes && n.HasPairs() {
if 2+index > lenBytes { if 2+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 6) return index, fmt.Errorf("index out of range %d", 6)
} }
n.PairsSize = util.BytesToUint16(bytes[index : index+2]) n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2 index += 2
if int(n.PairsSize)+index > lenBytes { if int(n.PairsSize)+index > lenBytes {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
return index, fmt.Errorf("index out of range %d", 7) return index, fmt.Errorf("index out of range %d", 7)
} }
end := index + int(n.PairsSize) end := index + int(n.PairsSize)

2
weed/storage/needle/needle_write.go
View file

@ -127,7 +127,7 @@ func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset u
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
return return
} }
if offset >= MaxPossibleVolumeSize && n.Size.IsValid() { if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize) err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
return return
} }

2
weed/storage/store.go
View file

@ -149,7 +149,7 @@ func (s *Store) FindFreeLocation(diskType DiskType) (ret *DiskLocation) {
} }
currentFreeCount := location.MaxVolumeCount - int32(location.VolumesLen()) currentFreeCount := location.MaxVolumeCount - int32(location.VolumesLen())
currentFreeCount *= erasure_coding.DataShardsCount currentFreeCount *= erasure_coding.DataShardsCount
currentFreeCount -= int32(location.EcVolumesLen()) currentFreeCount -= int32(location.EcShardCount())
currentFreeCount /= erasure_coding.DataShardsCount currentFreeCount /= erasure_coding.DataShardsCount
if currentFreeCount > max { if currentFreeCount > max {
max = currentFreeCount max = currentFreeCount

13
weed/storage/volume_write.go
View file

@ -107,16 +107,9 @@ func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) {
func (v *Volume) syncWrite(n *needle.Needle, checkCookie bool) (offset uint64, size Size, isUnchanged bool, err error) { func (v *Volume) syncWrite(n *needle.Needle, checkCookie bool) (offset uint64, size Size, isUnchanged bool, err error) {
// glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String()) // glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
actualSize := needle.GetActualSize(Size(len(n.Data)), v.Version())
v.dataFileAccessLock.Lock() v.dataFileAccessLock.Lock()
defer v.dataFileAccessLock.Unlock() defer v.dataFileAccessLock.Unlock()
if MaxPossibleVolumeSize < v.nm.ContentSize()+uint64(actualSize) {
err = fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.nm.ContentSize())
return
}
return v.doWriteRequest(n, checkCookie) return v.doWriteRequest(n, checkCookie)
} }
@ -193,7 +186,6 @@ func (v *Volume) doWriteRequest(n *needle.Needle, checkCookie bool) (offset uint
func (v *Volume) syncDelete(n *needle.Needle) (Size, error) { func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
// glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String()) // glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
actualSize := needle.GetActualSize(0, v.Version())
v.dataFileAccessLock.Lock() v.dataFileAccessLock.Lock()
defer v.dataFileAccessLock.Unlock() defer v.dataFileAccessLock.Unlock()
@ -201,11 +193,6 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
return 0, nil return 0, nil
} }
if MaxPossibleVolumeSize < v.nm.ContentSize()+uint64(actualSize) {
err := fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.nm.ContentSize())
return 0, err
}
return v.doDeleteRequest(n) return v.doDeleteRequest(n)
} }

6
weed/topology/store_replicate.go
View file

@ -48,7 +48,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync) isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds()) stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds())
if err != nil { if err != nil {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
err = fmt.Errorf("failed to write to local disk: %v", err) err = fmt.Errorf("failed to write to local disk: %v", err)
glog.V(0).Infoln(err) glog.V(0).Infoln(err)
return return
@ -80,7 +80,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
tmpMap := make(map[string]string) tmpMap := make(map[string]string)
err := json.Unmarshal(n.Pairs, &tmpMap) err := json.Unmarshal(n.Pairs, &tmpMap)
if err != nil { if err != nil {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
glog.V(0).Infoln("Unmarshal pairs error:", err) glog.V(0).Infoln("Unmarshal pairs error:", err)
} }
for k, v := range tmpMap { for k, v := range tmpMap {
@ -109,7 +109,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
}) })
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds()) stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds())
if err != nil { if err != nil {
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
glog.V(0).Infoln(err) glog.V(0).Infoln(err)
return false, err return false, err