mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'master' into mq-subscribe
This commit is contained in:
chrislu
commit
70a011a224
8
.github/workflows/binaries_dev.yml
vendored
8
.github/workflows/binaries_dev.yml
vendored
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -60,7 +60,7 @@ jobs:
|
|||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -93,7 +93,7 @@ jobs:
|
|||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -109,7 +109,7 @@ jobs:
|
|||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
4
.github/workflows/binaries_release0.yml
vendored
4
.github/workflows/binaries_release0.yml
vendored
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
4
.github/workflows/binaries_release1.yml
vendored
4
.github/workflows/binaries_release1.yml
vendored
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
4
.github/workflows/binaries_release2.yml
vendored
4
.github/workflows/binaries_release2.yml
vendored
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
4
.github/workflows/binaries_release3.yml
vendored
4
.github/workflows/binaries_release3.yml
vendored
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
4
.github/workflows/binaries_release4.yml
vendored
4
.github/workflows/binaries_release4.yml
vendored
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
@ -45,7 +45,7 @@ jobs:
|
|||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
|
||||
uses: wangyoucao577/go-release-action@af4a9db7b0ee3cf602cb75541d72bf568a99da4f # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
|
|
|||
2
.github/workflows/container_dev.yml
vendored
2
.github/workflows/container_dev.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_latest.yml
vendored
2
.github/workflows/container_latest.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release1.yml
vendored
2
.github/workflows/container_release1.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release2.yml
vendored
2
.github/workflows/container_release2.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
2
.github/workflows/container_release3.yml
vendored
2
.github/workflows/container_release3.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
|
|||
4
.github/workflows/container_release4.yml
vendored
4
.github/workflows/container_release4.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=elastic,ydb,gocdk,tikv
|
||||
build-args: TAGS=elastic,ydb,gocdk,tikv,rclone
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
|
|
|
|||
4
.github/workflows/container_release5.yml
vendored
4
.github/workflows/container_release5.yml
vendored
|
|
@ -20,7 +20,7 @@ jobs:
|
|||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
|
||||
uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv
|
||||
build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv,rclone
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
|
|
|
|||
2
.github/workflows/depsreview.yml
vendored
2
.github/workflows/depsreview.yml
vendored
|
|
@ -11,4 +11,4 @@ jobs:
|
|||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4
|
||||
uses: actions/dependency-review-action@c74b580d73376b7750d3d2a50bfb8adc2c937507
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -15,7 +15,7 @@ full_install:
|
|||
cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
|
||||
|
||||
server: install
|
||||
weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json
|
||||
weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324
|
||||
|
||||
benchmark: install warp_install
|
||||
pkill weed || true
|
||||
|
|
|
|||
26
go.mod
26
go.mod
|
|
@ -111,7 +111,7 @@ require (
|
|||
gocloud.dev/pubsub/rabbitpubsub v0.36.0
|
||||
golang.org/x/crypto v0.17.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
|
||||
golang.org/x/image v0.14.0
|
||||
golang.org/x/image v0.15.0
|
||||
golang.org/x/net v0.19.0
|
||||
golang.org/x/oauth2 v0.14.0 // indirect
|
||||
golang.org/x/sys v0.15.0
|
||||
|
|
@ -121,7 +121,7 @@ require (
|
|||
google.golang.org/api v0.151.0
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
|
||||
google.golang.org/grpc v1.59.0
|
||||
google.golang.org/grpc v1.60.1
|
||||
google.golang.org/protobuf v1.32.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
|
|
@ -140,9 +140,9 @@ require (
|
|||
github.com/Jille/raft-grpc-transport v1.5.0
|
||||
github.com/arangodb/go-driver v1.6.1
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/golang-jwt/jwt/v5 v5.1.0
|
||||
|
|
@ -188,26 +188,26 @@ require (
|
|||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
|
||||
github.com/aws/smithy-go v1.19.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.13.1 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/cloudflare/circl v1.3.6 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
|
|
@ -226,7 +226,7 @@ require (
|
|||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.10 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.10.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.11.0 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
|
|
|
|||
52
go.sum
52
go.sum
|
|
@ -141,20 +141,20 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
|
|||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY=
|
||||
github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw=
|
||||
|
|
@ -163,8 +163,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3
|
|||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY=
|
||||
|
|
@ -173,12 +173,12 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 h1:umyC9zH/A1w8AXrrG7iMxT4Rfgj8
|
|||
github.com/aws/aws-sdk-go-v2/service/sns v1.26.5/go.mod h1:IrcbquqMupzndZ20BXxDxjM7XenTRhbwBOetk4+Z5oc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 h1:cJb4I498c1mrOVrRqYTcnLD65AFqUuseHfzHdNZHL9U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5/go.mod h1:mCUv04gd/7g+/HNzDB4X6dzJuygji0ckvB3Lg/TdG5Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
|
||||
github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
|
||||
github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
|
|
@ -221,8 +221,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
|
|||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
|
||||
github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
|
||||
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 h1:KzZU0EMkUm4vX/jPp5d/VttocDpocL/8QP0zyiI9Xiw=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67/go.mod h1:xFxVVe3plxwhM+6BgTTPByEgG8hggo8+gtRUkbc5W8Q=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
|
|
@ -361,8 +361,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
|||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4+R14N6k=
|
||||
github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc=
|
||||
github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
|
||||
github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
|
||||
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
|
||||
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
|
||||
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
|
||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
|
|
@ -1073,8 +1073,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQz
|
|||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
|
||||
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
|
||||
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
@ -1467,8 +1467,8 @@ google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
|
|||
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
|
||||
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
|
||||
google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
|
||||
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
|
||||
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
|
||||
google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=
|
||||
google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
|
||||
google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk=
|
||||
google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE=
|
||||
|
|
|
|||
|
|
@ -121,10 +121,10 @@ func StreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, w
|
|||
remaining -= int64(chunkView.ViewSize)
|
||||
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
|
||||
return fmt.Errorf("read chunk: %v", err)
|
||||
}
|
||||
stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues("chunkDownload").Inc()
|
||||
downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize))
|
||||
}
|
||||
if remaining > 0 {
|
||||
|
|
|
|||
|
|
@ -329,7 +329,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
|
|||
if strings.Contains(post_err.Error(), "connection reset by peer") ||
|
||||
strings.Contains(post_err.Error(), "use of closed network connection") {
|
||||
glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr)
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc()
|
||||
resp, post_err = HttpClient.Do(req)
|
||||
defer util.CloseResponse(resp)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,29 +8,11 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
type StatusRecorder struct {
|
||||
http.ResponseWriter
|
||||
Status int
|
||||
}
|
||||
|
||||
func NewStatusResponseWriter(w http.ResponseWriter) *StatusRecorder {
|
||||
return &StatusRecorder{w, http.StatusOK}
|
||||
}
|
||||
|
||||
func (r *StatusRecorder) WriteHeader(status int) {
|
||||
r.Status = status
|
||||
r.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (r *StatusRecorder) Flush() {
|
||||
r.ResponseWriter.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
func track(f http.HandlerFunc, action string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
w.Header().Set("Server", "SeaweedFS S3")
|
||||
recorder := NewStatusResponseWriter(w)
|
||||
recorder := stats_collect.NewStatusResponseWriter(w)
|
||||
start := time.Now()
|
||||
f(recorder, r)
|
||||
if recorder.Status == http.StatusForbidden {
|
||||
|
|
|
|||
|
|
@ -111,11 +111,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
|
|||
|
||||
v.SetDefault("cors.allowed_origins.values", "*")
|
||||
|
||||
if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) {
|
||||
allowedOrigins := v.GetString("cors.allowed_origins.values")
|
||||
domains := strings.Split(allowedOrigins, ",")
|
||||
option.AllowedOrigins = domains
|
||||
}
|
||||
allowedOrigins := v.GetString("cors.allowed_origins.values")
|
||||
domains := strings.Split(allowedOrigins, ",")
|
||||
option.AllowedOrigins = domains
|
||||
|
||||
fs = &FilerServer{
|
||||
option: option,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
|
@ -17,7 +18,8 @@ import (
|
|||
|
||||
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
statusRecorder := stats.NewStatusResponseWriter(w)
|
||||
w = statusRecorder
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin != "" {
|
||||
if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" {
|
||||
|
|
@ -53,23 +55,23 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
|||
fileId = r.RequestURI[len("/?proxyChunkId="):]
|
||||
}
|
||||
if fileId != "" {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkProxy).Inc()
|
||||
fs.proxyToVolumeServer(w, r, fileId)
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkProxy).Inc()
|
||||
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkProxy).Observe(time.Since(start).Seconds())
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
stats.FilerRequestCounter.WithLabelValues(r.Method, strconv.Itoa(statusRecorder.Status)).Inc()
|
||||
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
|
||||
isReadHttpCall := r.Method == "GET" || r.Method == "HEAD"
|
||||
if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) {
|
||||
writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
|
||||
return
|
||||
}
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
|
||||
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
|
||||
|
||||
switch r.Method {
|
||||
|
|
@ -115,6 +117,8 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
|||
func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
start := time.Now()
|
||||
statusRecorder := stats.NewStatusResponseWriter(w)
|
||||
w = statusRecorder
|
||||
|
||||
os.Stdout.WriteString("Request: " + r.Method + " " + r.URL.String() + "\n")
|
||||
|
||||
|
|
@ -140,8 +144,8 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
|
|||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
|
||||
defer func() {
|
||||
stats.FilerRequestCounter.WithLabelValues(r.Method, strconv.Itoa(statusRecorder.Status)).Inc()
|
||||
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
// We handle OPTIONS first because it never should be authenticated
|
||||
|
|
|
|||
|
|
@ -98,11 +98,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
|||
}
|
||||
if err == filer_pb.ErrNotFound {
|
||||
glog.V(2).Infof("Not found %s: %v", path, err)
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadNotFound).Inc()
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
} else {
|
||||
glog.Errorf("Internal %s: %v", path, err)
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadInternal).Inc()
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
|
|
@ -233,7 +233,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
|||
if offset+size <= int64(len(entry.Content)) {
|
||||
_, err := writer.Write(entry.Content[offset : offset+size])
|
||||
if err != nil {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ErrorWriteEntry).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorWriteEntry).Inc()
|
||||
glog.Errorf("failed to write entry content: %v", err)
|
||||
}
|
||||
return err
|
||||
|
|
@ -245,7 +245,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
|||
Directory: dir,
|
||||
Name: name,
|
||||
}); err != nil {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadCache).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadCache).Inc()
|
||||
glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
|
||||
return fmt.Errorf("cache %s: %v", entry.FullPath, err)
|
||||
} else {
|
||||
|
|
@ -255,7 +255,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
err = filer.StreamContentWithThrottler(fs.filer.MasterClient, writer, chunks, offset, size, fs.option.DownloadMaxBytesPs)
|
||||
if err != nil {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadStream).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadStream).Inc()
|
||||
glog.Errorf("failed to stream content %s: %v", r.URL, err)
|
||||
}
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
// is empty.
|
||||
func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.DirList).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.DirList).Inc()
|
||||
|
||||
path := r.URL.Path
|
||||
if strings.HasSuffix(path, "/") && len(path) > 1 {
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ type FilerPostResult struct {
|
|||
|
||||
func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssign).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssign).Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkAssign).Observe(time.Since(start).Seconds())
|
||||
|
|
|
|||
|
|
@ -86,11 +86,11 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
|
|||
bytesBuffer.Read(smallContent)
|
||||
bufPool.Put(bytesBuffer)
|
||||
<-bytesBufferLimitChan
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ContentSaveToFiler).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ContentSaveToFiler).Inc()
|
||||
break
|
||||
}
|
||||
} else {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.AutoChunk).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.AutoChunk).Inc()
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
|
|
@ -143,7 +143,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
|
|||
|
||||
func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkUpload).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUpload).Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues(stats.ChunkUpload).Observe(time.Since(start).Seconds())
|
||||
|
|
@ -160,7 +160,7 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
|
|||
}
|
||||
uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
|
||||
if uploadResult != nil && uploadResult.RetryCount > 0 {
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount))
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount))
|
||||
}
|
||||
return uploadResult, err, data
|
||||
}
|
||||
|
|
@ -180,14 +180,14 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
|
|||
fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so)
|
||||
if uploadErr != nil {
|
||||
glog.V(4).Infof("retry later due to assign error: %v", uploadErr)
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkAssignRetry).Inc()
|
||||
return uploadErr
|
||||
}
|
||||
// upload the chunk to the volume server
|
||||
uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth)
|
||||
if uploadErr != nil {
|
||||
glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
|
||||
stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()
|
||||
fid, _ := filer_pb.ToFileIdObject(fileId)
|
||||
fileChunk := filer_pb.FileChunk{
|
||||
FileId: fileId,
|
||||
|
|
|
|||
|
|
@ -168,10 +168,10 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
|||
DataCenter: dn.GetDataCenterId(),
|
||||
}
|
||||
if len(heartbeat.NewVolumes) > 0 {
|
||||
stats.FilerRequestCounter.WithLabelValues("newVolumes").Inc()
|
||||
stats.MasterReceivedHeartbeatCounter.WithLabelValues("newVolumes").Inc()
|
||||
}
|
||||
if len(heartbeat.DeletedVolumes) > 0 {
|
||||
stats.FilerRequestCounter.WithLabelValues("deletedVolumes").Inc()
|
||||
stats.MasterReceivedHeartbeatCounter.WithLabelValues("deletedVolumes").Inc()
|
||||
}
|
||||
if len(heartbeat.NewVolumes) > 0 || len(heartbeat.DeletedVolumes) > 0 {
|
||||
// process delta volume ids if exists for fast volume id updates
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ security settings:
|
|||
*/
|
||||
|
||||
func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Request) {
|
||||
statusRecorder := stats.NewStatusResponseWriter(w)
|
||||
w = statusRecorder
|
||||
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
|
@ -38,10 +40,10 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
start := time.Now()
|
||||
requestMethod := r.Method
|
||||
defer func(start time.Time, method *string) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(*method).Inc()
|
||||
defer func(start time.Time, method *string, statusRecorder *stats.StatusRecorder) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
|
||||
}(start, &requestMethod)
|
||||
}(start, &requestMethod, statusRecorder)
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead:
|
||||
stats.ReadRequest()
|
||||
|
|
@ -63,11 +65,9 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
|||
vs.inFlightDownloadDataLimitCond.L.Unlock()
|
||||
vs.GetOrHeadHandler(w, r)
|
||||
case http.MethodDelete:
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
|
||||
stats.DeleteRequest()
|
||||
vs.guard.WhiteList(vs.DeleteHandler)(w, r)
|
||||
case http.MethodPut, http.MethodPost:
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
|
||||
contentLength := getContentLength(r)
|
||||
// exclude the replication from the concurrentUploadLimitMB
|
||||
if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
|
||||
|
|
@ -124,11 +124,21 @@ func getContentLength(r *http.Request) int64 {
|
|||
}
|
||||
|
||||
func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
statusRecorder := stats.NewStatusResponseWriter(w)
|
||||
w = statusRecorder
|
||||
w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION)
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
requestMethod := r.Method
|
||||
defer func(start time.Time, method *string, statusRecorder *stats.StatusRecorder) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(*method, strconv.Itoa(statusRecorder.Status)).Inc()
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
|
||||
}(start, &requestMethod, statusRecorder)
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead:
|
||||
stats.ReadRequest()
|
||||
|
|
|
|||
|
|
@ -30,12 +30,12 @@ import (
|
|||
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||
|
||||
func NotFound(w http.ResponseWriter) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorGetNotFound).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetNotFound).Inc()
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
|
||||
func InternalError(w http.ResponseWriter) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorGetInternal).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorGetInternal).Inc()
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
|
|
|
|||
21
weed/stats/http_status_recorder.go
Normal file
21
weed/stats/http_status_recorder.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
package stats
|
||||
|
||||
import "net/http"
|
||||
|
||||
type StatusRecorder struct {
|
||||
http.ResponseWriter
|
||||
Status int
|
||||
}
|
||||
|
||||
func NewStatusResponseWriter(w http.ResponseWriter) *StatusRecorder {
|
||||
return &StatusRecorder{w, http.StatusOK}
|
||||
}
|
||||
|
||||
func (r *StatusRecorder) WriteHeader(status int) {
|
||||
r.Status = status
|
||||
r.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (r *StatusRecorder) Flush() {
|
||||
r.ResponseWriter.(http.Flusher).Flush()
|
||||
}
|
||||
|
|
@ -84,6 +84,14 @@ var (
|
|||
Subsystem: "filer",
|
||||
Name: "request_total",
|
||||
Help: "Counter of filer requests.",
|
||||
}, []string{"type", "code"})
|
||||
|
||||
FilerHandlerCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: "filer",
|
||||
Name: "handler_total",
|
||||
Help: "Counter of filer handlers.",
|
||||
}, []string{"type"})
|
||||
|
||||
FilerRequestHistogram = prometheus.NewHistogramVec(
|
||||
|
|
@ -134,6 +142,14 @@ var (
|
|||
Subsystem: "volumeServer",
|
||||
Name: "request_total",
|
||||
Help: "Counter of volume server requests.",
|
||||
}, []string{"type", "code"})
|
||||
|
||||
VolumeServerHandlerCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: "volumeServer",
|
||||
Name: "handler_total",
|
||||
Help: "Counter of volume server handlers.",
|
||||
}, []string{"type"})
|
||||
|
||||
VolumeServerVacuumingCompactCounter = prometheus.NewCounterVec(
|
||||
|
|
@ -245,6 +261,7 @@ func init() {
|
|||
Gather.MustRegister(MasterReplicaPlacementMismatch)
|
||||
|
||||
Gather.MustRegister(FilerRequestCounter)
|
||||
Gather.MustRegister(FilerHandlerCounter)
|
||||
Gather.MustRegister(FilerRequestHistogram)
|
||||
Gather.MustRegister(FilerStoreCounter)
|
||||
Gather.MustRegister(FilerStoreHistogram)
|
||||
|
|
@ -254,6 +271,7 @@ func init() {
|
|||
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||
|
||||
Gather.MustRegister(VolumeServerRequestCounter)
|
||||
Gather.MustRegister(VolumeServerHandlerCounter)
|
||||
Gather.MustRegister(VolumeServerRequestHistogram)
|
||||
Gather.MustRegister(VolumeServerVacuumingCompactCounter)
|
||||
Gather.MustRegister(VolumeServerVacuumingCommitCounter)
|
||||
|
|
|
|||
|
|
@ -218,9 +218,13 @@ func (l *DiskLocation) unmountEcVolumeByCollection(collectionName string) map[ne
|
|||
return deltaVols
|
||||
}
|
||||
|
||||
func (l *DiskLocation) EcVolumesLen() int {
|
||||
func (l *DiskLocation) EcShardCount() int {
|
||||
l.ecVolumesLock.RLock()
|
||||
defer l.ecVolumesLock.RUnlock()
|
||||
|
||||
return len(l.ecVolumes)
|
||||
shardCount := 0
|
||||
for _, ecVolume := range l.ecVolumes {
|
||||
shardCount += len(ecVolume.Shards)
|
||||
}
|
||||
return shardCount
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,11 +54,11 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
|
|||
if n.Size != size {
|
||||
// cookie is not always passed in for this API. Use size to do preliminary checking.
|
||||
if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc()
|
||||
glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
|
||||
return ErrorSizeMismatch
|
||||
}
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc()
|
||||
return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
|
||||
}
|
||||
switch version {
|
||||
|
|
@ -75,7 +75,7 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Versio
|
|||
newChecksum := NewCRC(n.Data)
|
||||
if checksum != newChecksum.Value() && checksum != uint32(newChecksum) {
|
||||
// the crc.Value() function is to be deprecated. this double checking is for backward compatible.
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorCRC).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc()
|
||||
return errors.New("CRC error! Data On Disk Corrupted")
|
||||
}
|
||||
n.Checksum = newChecksum
|
||||
|
|
@ -108,7 +108,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
|
|||
n.DataSize = util.BytesToUint32(bytes[index : index+4])
|
||||
index = index + 4
|
||||
if int(n.DataSize)+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return fmt.Errorf("index out of range %d", 1)
|
||||
}
|
||||
n.Data = bytes[index : index+int(n.DataSize)]
|
||||
|
|
@ -127,7 +127,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
|
|||
n.NameSize = uint8(bytes[index])
|
||||
index = index + 1
|
||||
if int(n.NameSize)+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 2)
|
||||
}
|
||||
n.Name = bytes[index : index+int(n.NameSize)]
|
||||
|
|
@ -137,7 +137,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
|
|||
n.MimeSize = uint8(bytes[index])
|
||||
index = index + 1
|
||||
if int(n.MimeSize)+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 3)
|
||||
}
|
||||
n.Mime = bytes[index : index+int(n.MimeSize)]
|
||||
|
|
@ -145,7 +145,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
|
|||
}
|
||||
if index < lenBytes && n.HasLastModifiedDate() {
|
||||
if LastModifiedBytesLength+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 4)
|
||||
}
|
||||
n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
|
||||
|
|
@ -153,7 +153,7 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
|
|||
}
|
||||
if index < lenBytes && n.HasTtl() {
|
||||
if TtlBytesLength+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 5)
|
||||
}
|
||||
n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
|
||||
|
|
@ -161,13 +161,13 @@ func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err err
|
|||
}
|
||||
if index < lenBytes && n.HasPairs() {
|
||||
if 2+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 6)
|
||||
}
|
||||
n.PairsSize = util.BytesToUint16(bytes[index : index+2])
|
||||
index += 2
|
||||
if int(n.PairsSize)+index > lenBytes {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
|
||||
return index, fmt.Errorf("index out of range %d", 7)
|
||||
}
|
||||
end := index + int(n.PairsSize)
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset u
|
|||
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
|
||||
return
|
||||
}
|
||||
if offset >= MaxPossibleVolumeSize && n.Size.IsValid() {
|
||||
if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
|
||||
err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ func (s *Store) FindFreeLocation(diskType DiskType) (ret *DiskLocation) {
|
|||
}
|
||||
currentFreeCount := location.MaxVolumeCount - int32(location.VolumesLen())
|
||||
currentFreeCount *= erasure_coding.DataShardsCount
|
||||
currentFreeCount -= int32(location.EcVolumesLen())
|
||||
currentFreeCount -= int32(location.EcShardCount())
|
||||
currentFreeCount /= erasure_coding.DataShardsCount
|
||||
if currentFreeCount > max {
|
||||
max = currentFreeCount
|
||||
|
|
|
|||
|
|
@ -107,16 +107,9 @@ func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) {
|
|||
|
||||
func (v *Volume) syncWrite(n *needle.Needle, checkCookie bool) (offset uint64, size Size, isUnchanged bool, err error) {
|
||||
// glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
|
||||
actualSize := needle.GetActualSize(Size(len(n.Data)), v.Version())
|
||||
|
||||
v.dataFileAccessLock.Lock()
|
||||
defer v.dataFileAccessLock.Unlock()
|
||||
|
||||
if MaxPossibleVolumeSize < v.nm.ContentSize()+uint64(actualSize) {
|
||||
err = fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.nm.ContentSize())
|
||||
return
|
||||
}
|
||||
|
||||
return v.doWriteRequest(n, checkCookie)
|
||||
}
|
||||
|
||||
|
|
@ -193,7 +186,6 @@ func (v *Volume) doWriteRequest(n *needle.Needle, checkCookie bool) (offset uint
|
|||
|
||||
func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
|
||||
// glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
|
||||
actualSize := needle.GetActualSize(0, v.Version())
|
||||
v.dataFileAccessLock.Lock()
|
||||
defer v.dataFileAccessLock.Unlock()
|
||||
|
||||
|
|
@ -201,11 +193,6 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
if MaxPossibleVolumeSize < v.nm.ContentSize()+uint64(actualSize) {
|
||||
err := fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.nm.ContentSize())
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return v.doDeleteRequest(n)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
|
|||
isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
|
||||
err = fmt.Errorf("failed to write to local disk: %v", err)
|
||||
glog.V(0).Infoln(err)
|
||||
return
|
||||
|
|
@ -80,7 +80,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
|
|||
tmpMap := make(map[string]string)
|
||||
err := json.Unmarshal(n.Pairs, &tmpMap)
|
||||
if err != nil {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
|
||||
glog.V(0).Infoln("Unmarshal pairs error:", err)
|
||||
}
|
||||
for k, v := range tmpMap {
|
||||
|
|
@ -109,7 +109,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
|
|||
})
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
|
||||
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
|
||||
glog.V(0).Infoln(err)
|
||||
return false, err
|
||||
|
|
|
|||
Loading…
Reference in a new issue