From 0bb97709d41b1be4c74f01dcc65aac6d5f88bd16 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 18 Sep 2023 16:13:20 -0700 Subject: [PATCH] Revert "Merge branch 'master' into sub" This reverts commit 4d414f54a224142f3f4d934f4af3b5dceb6fec6b, reversing changes made to 482742514656e9b5a652acf7406740fbc55db13d. --- .github/workflows/container_dev.yml | 12 ++-- .github/workflows/container_latest.yml | 12 ++-- .github/workflows/container_release1.yml | 10 +-- .github/workflows/container_release2.yml | 10 +-- .github/workflows/container_release3.yml | 10 +-- .github/workflows/container_release4.yml | 10 +-- .github/workflows/container_release5.yml | 10 +-- go.mod | 39 +++++------ go.sum | 73 ++++++++++--------- weed/cluster/lock_manager/lock_manager.go | 2 +- weed/mount/dirty_pages_chunked.go | 7 +- weed/mq/balancer/allocate.go | 20 ------ weed/mq/balancer/allocate_test.go | 62 ----------------- weed/mq/balancer/balancer.go | 59 +--------------- weed/mq/balancer/lookup.go | 43 ------------ weed/mq/broker/broker_grpc_admin.go | 4 ++ weed/mq/topic_allocation/allocation.go | 81 ++++++++++++++++++++++ weed/server/filer_server.go | 7 +- weed/server/filer_server_handlers.go | 12 ++-- weed/server/filer_server_handlers_write.go | 2 +- weed/storage/store_ec_delete.go | 4 +- 21 files changed, 199 insertions(+), 290 deletions(-) delete mode 100644 weed/mq/balancer/allocate.go delete mode 100644 weed/mq/balancer/allocate_test.go delete mode 100644 weed/mq/balancer/lookup.go create mode 100644 weed/mq/topic_allocation/allocation.go diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 212fabd24..23d6bbc1b 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -33,30 +33,30 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 with: buildkitd-flags: "--debug" - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index faace8bf5..99f18941f 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -34,30 +34,30 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 with: buildkitd-flags: "--debug" - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index 7bc023d39..1046880ee 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index deb953a57..385c72f2e 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -35,20 +35,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 8cfe2ffec..e8713ffcb 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -35,20 +35,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index d19ce3e43..199652ad3 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index d989f4b04..31c9384a0 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3 + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3 with: images: | chrislusf/seaweedfs @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2 + uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/go.mod b/go.mod index a500cbda3..798ec0d70 100644 --- a/go.mod +++ b/go.mod @@ -112,18 +112,18 @@ require ( gocloud.dev v0.34.0 gocloud.dev/pubsub/natspubsub v0.33.0 gocloud.dev/pubsub/rabbitpubsub v0.33.0 - golang.org/x/crypto v0.13.0 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/image v0.11.0 - golang.org/x/net v0.15.0 - golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.12.0 - golang.org/x/text v0.13.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/tools v0.11.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.141.0 + google.golang.org/api v0.134.0 google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect google.golang.org/grpc v1.58.0 google.golang.org/protobuf v1.31.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -131,7 +131,7 @@ require ( modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect modernc.org/libc v1.24.1 // indirect - modernc.org/mathutil v1.6.0 // indirect + modernc.org/mathutil v1.6.0 modernc.org/memory v1.6.0 // indirect modernc.org/opt v0.1.3 // indirect modernc.org/sqlite v1.25.0 @@ -146,20 +146,18 @@ require ( github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.39 github.com/aws/aws-sdk-go-v2/credentials v1.13.37 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 github.com/fluent/fluent-logger-golang v1.9.0 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 - github.com/hanwen/go-fuse/v2 v2.4.0 + github.com/hanwen/go-fuse/v2 v2.3.0 github.com/hashicorp/raft v1.5.0 github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/puzpuzpuz/xsync/v2 v2.5.0 github.com/rabbitmq/amqp091-go v1.8.1 github.com/rclone/rclone v1.63.1 - github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 github.com/schollz/progressbar/v3 v3.13.1 - github.com/shirou/gopsutil/v3 v3.23.5 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8 @@ -181,16 +179,16 @@ require ( github.com/Unknwon/goconfig v1.0.0 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect @@ -217,7 +215,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect @@ -271,6 +269,7 @@ require ( github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/shirou/gopsutil/v3 v3.23.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/smartystreets/goconvey v1.8.1 // indirect @@ -303,10 +302,10 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/term v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 23bf14320..7249eb68f 100644 --- a/go.sum +++ b/go.sum @@ -114,10 +114,11 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+ github.com/aws/aws-sdk-go v1.44.314 h1:d/5Jyk/Fb+PBd/4nzQg0JuC2W4A0knrDIzBgK/ggAow= github.com/aws/aws-sdk-go v1.44.314/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= +github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkMTXxTs0oLBGBKpBZbk3NLh8EvAfF0Q3x8/0c= github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ= github.com/aws/aws-sdk-go-v2/config v1.18.39/go.mod h1:+NH/ZigdPckFpgB1TRcRuWCB/Kbbvkxc/iNAKTq5RhE= github.com/aws/aws-sdk-go-v2/credentials v1.13.37 h1:BvEdm09+ZEh2XtN+PVHPcYwKY3wIeB6pw7vPRM4M9/U= @@ -125,25 +126,28 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.37/go.mod h1:ACLrdkd4CLZyXOghZ8IY github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 h1:vUh7dBFNS3oFCtVv6CiYKh5hP9ls8+kIpKLeFruIBLk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1/go.mod h1:sFMeinkhj/SZKQM8BxtvNtSPjJEo0Xrz+w3g2e4FSKI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 h1:iV/W5OMBys+66OeXJi/7xIRrKZNsu0ylsLGu+6nbmQE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13/go.mod h1:ReJb6xYmtGyu9KoFtRreWegbN9dZqvZIIv4vWnhcsyI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 h1:QviNkc+vGSuEHx8P+pVNKOdWLXBPIwMFv7p0fphgE4U= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33/go.mod h1:fABTUmOrAgAalG2i9WJpjBvlnk7UK8YmnYaxN+Q2CwE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 h1:PT6PBCycRwhpEW5hJnRiceCeoWJ+r3bdgXtV+VKG7Pk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1/go.mod h1:TqoxCLwT2nrxrBGA+z7t6OWM7LBkgRckK3gOjYE+7JA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 h1:v346f1h8sUBKXnEbrv43L37MTBlFHyKXQPIZHNAaghA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2/go.mod h1:cwCATiyNrXK9P2FsWdZ89g9mpsYv2rhk0UA/KByl5fY= github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 h1:Q01Dph/7FaB41Z7EY+SoVPa/kMpLGFiQPmF2PpVzaCE= github.com/aws/aws-sdk-go-v2/service/sns v1.21.1/go.mod h1:laHbYFVzphXdCiT3gitfuCDA2Oukrt9p40jWK7OJLgc= github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8= @@ -155,6 +159,7 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6/go.mod h1:yygr8ACQRY2PrEcy3 github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -412,8 +417,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -441,8 +446,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hanwen/go-fuse/v2 v2.4.0 h1:12OhD7CkXXQdvxG2osIdBQLdXh+nmLXY9unkUIe/xaU= -github.com/hanwen/go-fuse/v2 v2.4.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= +github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= +github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -979,12 +984,13 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1086,8 +1092,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1097,8 +1103,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1207,8 +1213,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1223,9 +1229,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1322,8 +1327,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.141.0 h1:Df6vfMgDoIM6ss0m7H4MPwFwY87WNXHfBIda/Bmfl4E= -google.golang.org/api v0.141.0/go.mod h1:iZqLkdPlXKyG0b90eu6KxVSE4D/ccRF2e/doKD2CnQQ= +google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= +google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1373,12 +1378,12 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 h1:o4LtQxebKIJ4vkzyhtD2rfUNZ20Zf0ik5YVP5E7G7VE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf h1:v5Cf4E9+6tawYrs/grq1q1hFpGtzlGFzgWHqwt6NFiU= +google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf h1:xkVZ5FdZJF4U82Q/JS+DcZA83s/GRVL+QrFMlexk9Yo= +google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf h1:guOdSPaeFgN+jEJwTo1dQ71hdBm+yKSCCKuTRkJzcVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/weed/cluster/lock_manager/lock_manager.go b/weed/cluster/lock_manager/lock_manager.go index 49b951dd9..0e3e47ba4 100644 --- a/weed/cluster/lock_manager/lock_manager.go +++ b/weed/cluster/lock_manager/lock_manager.go @@ -139,7 +139,7 @@ func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, func (lm *LockManager) GetLockOwner(key string) (owner string, err error) { lm.locks.Range(func(k string, lock *Lock) bool { - if k == key && lock != nil { + if k == key { owner = lock.Owner return false } diff --git a/weed/mount/dirty_pages_chunked.go b/weed/mount/dirty_pages_chunked.go index 25b071e7d..76a3be1be 100644 --- a/weed/mount/dirty_pages_chunked.go +++ b/weed/mount/dirty_pages_chunked.go @@ -2,12 +2,11 @@ package mount import ( "fmt" - "io" - "sync" - "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/mount/page_writer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "io" + "sync" ) type ChunkedDirtyPages struct { @@ -83,7 +82,7 @@ func (pages *ChunkedDirtyPages) saveChunkedFileIntervalToStorage(reader io.Reade } -func (pages *ChunkedDirtyPages) Destroy() { +func (pages ChunkedDirtyPages) Destroy() { pages.uploadPipeline.Shutdown() } diff --git a/weed/mq/balancer/allocate.go b/weed/mq/balancer/allocate.go deleted file mode 100644 index d594c60fb..000000000 --- a/weed/mq/balancer/allocate.go +++ /dev/null @@ -1,20 +0,0 @@ -package balancer - -import ( - cmap "github.com/orcaman/concurrent-map/v2" - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" -) - -func allocateTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStats], partitionCount int) (assignments []*mq_pb.BrokerPartitionAssignment) { - return []*mq_pb.BrokerPartitionAssignment{ - { - LeaderBroker: "localhost:17777", - FollowerBrokers: []string{"localhost:17777"}, - Partition: &mq_pb.Partition{ - RingSize: MaxPartitionCount, - RangeStart: 0, - RangeStop: MaxPartitionCount, - }, - }, - } -} diff --git a/weed/mq/balancer/allocate_test.go b/weed/mq/balancer/allocate_test.go deleted file mode 100644 index c714788e6..000000000 --- a/weed/mq/balancer/allocate_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package balancer - -import ( - cmap "github.com/orcaman/concurrent-map/v2" - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" - "reflect" - "testing" -) - -func Test_allocateOneBroker(t *testing.T) { - brokers := cmap.New[*BrokerStats]() - brokers.SetIfAbsent("localhost:17777", &BrokerStats{ - TopicPartitionCount: 0, - ConsumerCount: 0, - CpuUsagePercent: 0, - }) - - tests := []struct { - name string - args args - wantAssignments []*mq_pb.BrokerPartitionAssignment - }{ - { - name: "test only one broker", - args: args{ - brokers: brokers, - partitionCount: 6, - }, - wantAssignments: []*mq_pb.BrokerPartitionAssignment{ - { - LeaderBroker: "localhost:17777", - FollowerBrokers: []string{"localhost:17777"}, - Partition: &mq_pb.Partition{ - RingSize: MaxPartitionCount, - RangeStart: 0, - RangeStop: MaxPartitionCount, - }, - }, - }, - }, - } - testThem(t, tests) -} - -type args struct { - brokers cmap.ConcurrentMap[string, *BrokerStats] - partitionCount int -} - -func testThem(t *testing.T, tests []struct { - name string - args args - wantAssignments []*mq_pb.BrokerPartitionAssignment -}) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotAssignments := allocateTopicPartitions(tt.args.brokers, tt.args.partitionCount); !reflect.DeepEqual(gotAssignments, tt.wantAssignments) { - t.Errorf("allocateTopicPartitions() = %v, want %v", gotAssignments, tt.wantAssignments) - } - }) - } -} diff --git a/weed/mq/balancer/balancer.go b/weed/mq/balancer/balancer.go index 837dc0ce3..74871925f 100644 --- a/weed/mq/balancer/balancer.go +++ b/weed/mq/balancer/balancer.go @@ -1,67 +1,16 @@ package balancer import ( - "fmt" cmap "github.com/orcaman/concurrent-map/v2" - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" -) - -const ( - MaxPartitionCount = 8 * 9 * 5 * 7 //2520 ) type Balancer struct { Brokers cmap.ConcurrentMap[string, *BrokerStats] } - type BrokerStats struct { TopicPartitionCount int32 ConsumerCount int32 CpuUsagePercent int32 - Stats cmap.ConcurrentMap[string, *TopicPartitionStats] -} - -func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) { - bs.TopicPartitionCount = int32(len(stats.Stats)) - bs.CpuUsagePercent = stats.CpuUsagePercent - - var consumerCount int32 - currentTopicPartitions := bs.Stats.Items() - for _, topicPartitionStats := range stats.Stats { - tps := &TopicPartitionStats{ - TopicPartition: TopicPartition{ - Namespace: topicPartitionStats.Topic.Namespace, - Topic: topicPartitionStats.Topic.Name, - RangeStart: topicPartitionStats.Partition.RangeStart, - RangeStop: topicPartitionStats.Partition.RangeStop, - }, - ConsumerCount: topicPartitionStats.ConsumerCount, - IsLeader: topicPartitionStats.IsLeader, - } - consumerCount += topicPartitionStats.ConsumerCount - key := tps.TopicPartition.String() - bs.Stats.Set(key, tps) - delete(currentTopicPartitions, key) - } - // remove the topic partitions that are not in the stats - for key := range currentTopicPartitions { - bs.Stats.Remove(key) - } - bs.ConsumerCount = consumerCount - -} - -type TopicPartition struct { - Namespace string - Topic string - RangeStart int32 - RangeStop int32 -} - -type TopicPartitionStats struct { - TopicPartition - ConsumerCount int32 - IsLeader bool } func NewBalancer() *Balancer { @@ -71,11 +20,5 @@ func NewBalancer() *Balancer { } func NewBrokerStats() *BrokerStats { - return &BrokerStats{ - Stats: cmap.New[*TopicPartitionStats](), - } -} - -func (tp *TopicPartition) String() string { - return fmt.Sprintf("%v.%v-%04d-%04d", tp.Namespace, tp.Topic, tp.RangeStart, tp.RangeStop) + return &BrokerStats{} } diff --git a/weed/mq/balancer/lookup.go b/weed/mq/balancer/lookup.go deleted file mode 100644 index 55ed3e95d..000000000 --- a/weed/mq/balancer/lookup.go +++ /dev/null @@ -1,43 +0,0 @@ -package balancer - -import ( - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" -) - -func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish bool) (assignments []*mq_pb.BrokerPartitionAssignment, err error) { - // find existing topic partition assignments - for brokerStatsItem := range b.Brokers.IterBuffered() { - broker, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val - for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() { - topicPartitionStat := topicPartitionStatsItem.Val - if topicPartitionStat.TopicPartition.Namespace == topic.Namespace && - topicPartitionStat.TopicPartition.Topic == topic.Name { - assignment := &mq_pb.BrokerPartitionAssignment{ - Partition: &mq_pb.Partition{ - RingSize: MaxPartitionCount, - RangeStart: topicPartitionStat.RangeStart, - RangeStop: topicPartitionStat.RangeStop, - }, - } - if topicPartitionStat.IsLeader { - assignment.LeaderBroker = broker - } else { - assignment.FollowerBrokers = append(assignment.FollowerBrokers, broker) - } - assignments = append(assignments, assignment) - } - } - } - if len(assignments) > 0 { - return assignments, nil - } - - // find the topic partitions on the filer - // if the topic is not found - // if the request is_for_publish - // create the topic - // if the request is_for_subscribe - // return error not found - // t := topic.FromPbTopic(request.Topic) - return allocateTopicPartitions(b.Brokers, 6), nil -} diff --git a/weed/mq/broker/broker_grpc_admin.go b/weed/mq/broker/broker_grpc_admin.go index b24bf08a4..7337ba23e 100644 --- a/weed/mq/broker/broker_grpc_admin.go +++ b/weed/mq/broker/broker_grpc_admin.go @@ -12,6 +12,10 @@ import ( "sync" ) +const ( + MaxPartitionCount = 1024 +) + func (broker *MessageQueueBroker) FindBrokerLeader(c context.Context, request *mq_pb.FindBrokerLeaderRequest) (*mq_pb.FindBrokerLeaderResponse, error) { ret := &mq_pb.FindBrokerLeaderResponse{} err := broker.withMasterClient(false, broker.MasterClient.GetMaster(), func(client master_pb.SeaweedClient) error { diff --git a/weed/mq/topic_allocation/allocation.go b/weed/mq/topic_allocation/allocation.go new file mode 100644 index 000000000..a07ce4884 --- /dev/null +++ b/weed/mq/topic_allocation/allocation.go @@ -0,0 +1,81 @@ +package topic_allocation + +import ( + "github.com/seaweedfs/seaweedfs/weed/mq/topic" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" + "modernc.org/mathutil" +) + +const ( + DefaultBrokerCount = 4 +) + +// AllocateBrokersForTopicPartitions allocate brokers for a topic's all partitions +func AllocateBrokersForTopicPartitions(t topic.Topic, prevAssignment *mq_pb.TopicPartitionsAssignment, candidateBrokers []pb.ServerAddress) (assignment *mq_pb.TopicPartitionsAssignment, err error) { + // create a previous assignment if not exists + if prevAssignment == nil || len(prevAssignment.BrokerPartitions) == 0 { + prevAssignment = &mq_pb.TopicPartitionsAssignment{ + PartitionCount: topic.PartitionCount, + } + partitionCountForEachBroker := topic.PartitionCount / DefaultBrokerCount + for i := 0; i < DefaultBrokerCount; i++ { + prevAssignment.BrokerPartitions = append(prevAssignment.BrokerPartitions, &mq_pb.BrokerPartitionsAssignment{ + PartitionStart: int32(i * partitionCountForEachBroker), + PartitionStop: mathutil.MaxInt32(int32((i+1)*partitionCountForEachBroker), topic.PartitionCount), + }) + } + } + + // create a new assignment + assignment = &mq_pb.TopicPartitionsAssignment{ + PartitionCount: prevAssignment.PartitionCount, + } + + // allocate partitions for each partition range + for _, brokerPartition := range prevAssignment.BrokerPartitions { + // allocate partitions for each partition range + leader, followers, err := allocateBrokersForOneTopicPartition(t, brokerPartition, candidateBrokers) + if err != nil { + return nil, err + } + + followerBrokers := make([]string, len(followers)) + for i, follower := range followers { + followerBrokers[i] = string(follower) + } + + assignment.BrokerPartitions = append(assignment.BrokerPartitions, &mq_pb.BrokerPartitionsAssignment{ + PartitionStart: brokerPartition.PartitionStart, + PartitionStop: brokerPartition.PartitionStop, + LeaderBroker: string(leader), + FollowerBrokers: followerBrokers, + }) + } + + return +} + +func allocateBrokersForOneTopicPartition(t topic.Topic, brokerPartition *mq_pb.BrokerPartitionsAssignment, candidateBrokers []pb.ServerAddress) (leader pb.ServerAddress, followers []pb.ServerAddress, err error) { + // allocate leader + leader, err = allocateLeaderForOneTopicPartition(t, brokerPartition, candidateBrokers) + if err != nil { + return + } + + // allocate followers + followers, err = allocateFollowersForOneTopicPartition(t, brokerPartition, candidateBrokers) + if err != nil { + return + } + + return +} + +func allocateFollowersForOneTopicPartition(t topic.Topic, partition *mq_pb.BrokerPartitionsAssignment, brokers []pb.ServerAddress) (followers []pb.ServerAddress, err error) { + return +} + +func allocateLeaderForOneTopicPartition(t topic.Topic, partition *mq_pb.BrokerPartitionsAssignment, brokers []pb.ServerAddress) (leader pb.ServerAddress, err error) { + return +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 788ee150a..98784bce3 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -94,6 +94,9 @@ type FilerServer struct { // track known metadata listeners knownListenersLock sync.Mutex knownListeners map[int32]int32 + + // client to assign file id + assignProxy *operation.AssignProxy } func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { @@ -132,6 +135,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec) go fs.filer.KeepMasterClientConnected() + fs.assignProxy, err = operation.NewAssignProxy(fs.filer.GetMaster, fs.grpcDialOption, 16) + if !util.LoadConfiguration("filer", false) { v.SetDefault("leveldb2.enabled", true) v.SetDefault("leveldb2.dir", option.DefaultLevelDbDir) @@ -184,7 +189,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs.filer.Dlm.LockRing.SetTakeSnapshotCallback(fs.OnDlmChangeSnapshot) - return fs, nil + return fs, err } func (fs *FilerServer) checkWithMaster() { diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 54ddfb8b2..454148e66 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -30,6 +30,12 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { return } + isReadHttpCall := r.Method == "GET" || r.Method == "HEAD" + if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) { + writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) + return + } + // proxy to volume servers var fileId string if strings.HasPrefix(r.RequestURI, "/?proxyChunkId=") { @@ -42,12 +48,6 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { return } - isReadHttpCall := r.Method == "GET" || r.Method == "HEAD" - if !fs.maybeCheckJwtAuthorization(r, !isReadHttpCall) { - writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt")) - return - } - stats.FilerRequestCounter.WithLabelValues(r.Method).Inc() defer func() { stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds()) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 898975d14..daf63fa8d 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -43,7 +43,7 @@ func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, u ar, altRequest := so.ToAssignRequests(1) - assignResult, ae := operation.Assign(fs.filer.GetMaster, fs.grpcDialOption, ar, altRequest) + assignResult, ae := fs.assignProxy.Assign(ar, altRequest) if ae != nil { glog.Errorf("failing to assign a file id: %v", ae) err = ae diff --git a/weed/storage/store_ec_delete.go b/weed/storage/store_ec_delete.go index a3e028bbb..4b24cae79 100644 --- a/weed/storage/store_ec_delete.go +++ b/weed/storage/store_ec_delete.go @@ -36,9 +36,7 @@ func (s *Store) DeleteEcShardNeedle(ecVolume *erasure_coding.EcVolume, n *needle func (s *Store) doDeleteNeedleFromAtLeastOneRemoteEcShards(ecVolume *erasure_coding.EcVolume, needleId types.NeedleId) error { _, _, intervals, err := ecVolume.LocateEcShardNeedle(needleId, ecVolume.Version) - if err != nil { - return err - } + if len(intervals) == 0 { return erasure_coding.NotFoundError }