mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'master' into circuit_breaker
This commit is contained in:
LHHDZ
GitHub
commit
8db9f13bc6
2
.github/workflows/container_dev.yml
vendored
2
.github/workflows/container_dev.yml
vendored
|
|
@ -3,7 +3,7 @@ name: "docker: build dev containers"
|
|||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_latest.yml
vendored
2
.github/workflows/container_latest.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_release1.yml
vendored
2
.github/workflows/container_release1.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_release2.yml
vendored
2
.github/workflows/container_release2.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_release3.yml
vendored
2
.github/workflows/container_release3.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_release4.yml
vendored
2
.github/workflows/container_release4.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
2
.github/workflows/container_release5.yml
vendored
2
.github/workflows/container_release5.yml
vendored
|
|
@ -4,7 +4,7 @@ on:
|
|||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch: []
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
|
|||
25
go.mod
25
go.mod
|
|
@ -9,8 +9,8 @@ require (
|
|||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/Shopify/sarama v1.34.0
|
||||
github.com/aws/aws-sdk-go v1.44.14
|
||||
github.com/Shopify/sarama v1.34.1
|
||||
github.com/aws/aws-sdk-go v1.44.32
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
|
|
@ -64,11 +64,11 @@ require (
|
|||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.1 // indirect
|
||||
github.com/klauspost/compress v1.15.6 // indirect
|
||||
github.com/klauspost/reedsolomon v1.9.16
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.6
|
||||
github.com/linxGnu/grocksdb v1.7.2
|
||||
github.com/linxGnu/grocksdb v1.7.3
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.3 // indirect
|
||||
|
|
@ -123,16 +123,16 @@ require (
|
|||
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
|
||||
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
|
||||
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d
|
||||
golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
|
||||
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
|
||||
google.golang.org/api v0.81.0
|
||||
google.golang.org/api v0.83.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect
|
||||
google.golang.org/grpc v1.46.2
|
||||
google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
|
||||
google.golang.org/grpc v1.47.0
|
||||
google.golang.org/protobuf v1.28.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
|
|
@ -155,7 +155,7 @@ require (
|
|||
github.com/hashicorp/raft v1.3.9
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10
|
||||
)
|
||||
|
||||
require (
|
||||
|
|
@ -190,6 +190,7 @@ require (
|
|||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
|
|
@ -206,7 +207,7 @@ require (
|
|||
github.com/subosito/gotenv v1.3.0 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
|
||||
|
|
@ -215,7 +216,7 @@ require (
|
|||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
|
||||
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
|
||||
gopkg.in/ini.v1 v1.66.4 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
|
|
|||
44
go.sum
44
go.sum
|
|
@ -128,10 +128,10 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
|
|||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
|
||||
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/Shopify/sarama v1.34.0 h1:j4zTaFHFnfvuV2fdLZyXqIg0Tu4Mzl9f064Z5/H+o4o=
|
||||
github.com/Shopify/sarama v1.34.0/go.mod h1:V2ceE9UupUf4/oP1Z38SI49fAnD0/MtkqDDHvolIeeQ=
|
||||
github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ=
|
||||
github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c=
|
||||
github.com/Shopify/sarama v1.34.1 h1:pVCQO7BMAK3s1jWhgi5v1W6lwZ6Veiekfc2vsgRS06Y=
|
||||
github.com/Shopify/sarama v1.34.1/go.mod h1:NZSNswsnStpq8TUdFaqnpXm2Do6KRzTIjdBdVlL1YRM=
|
||||
github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64=
|
||||
github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
|
|
@ -150,8 +150,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
|
|||
github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
|
||||
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.44.14 h1:qd7/muV1rElsbvkK9D1nHUzBoDlEw2etfeo4IE82eSQ=
|
||||
github.com/aws/aws-sdk-go v1.44.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go v1.44.32 h1:x5hBtpY/02sgRL158zzTclcCLwh3dx3YlSl1rAH4Op0=
|
||||
github.com/aws/aws-sdk-go v1.44.32/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
|
||||
|
|
@ -579,6 +579,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
|
|||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
|
||||
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
|
|
@ -603,9 +605,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
|||
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
|
||||
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI=
|
||||
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/reedsolomon v1.9.16 h1:mR0AwphBwqFv/I3B9AHtNKvzuowI1vrj8/3UX4XRmHA=
|
||||
|
|
@ -636,8 +638,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
|||
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
|
||||
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/linxGnu/grocksdb v1.7.2 h1:5SVU++WAyOzk4jG0pXrI9plvZEPRCRbh4UrDhAxrktM=
|
||||
github.com/linxGnu/grocksdb v1.7.2/go.mod h1:3tKVazPI6orwnNtFAOSRZwhi6bMOkc3+kREb8qpKaSE=
|
||||
github.com/linxGnu/grocksdb v1.7.3 h1:S9XiU4FviunvjNdNG+kWe2BoOy/2EKZSdDyeGmL0vDs=
|
||||
github.com/linxGnu/grocksdb v1.7.3/go.mod h1:G4zrMNj2CP2aCXF61jbmZH81tu+kU3qU4rYpOU8WOL8=
|
||||
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
|
||||
github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
|
|
@ -745,6 +747,7 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
|
|||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
|
||||
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
|
|
@ -868,12 +871,14 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO
|
|||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8=
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 h1:S3bwscnat3pa188pvEuXDUstxPjCnGpn5bZJuRMkL+g=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3 h1:yyMw+sTDSqGVDG8ivAw8V/kXfaulXsgftSG+4nCmseA=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10 h1:2M6L2NX8l103qNDM1i97gh81Y2EJEAEyON3/N5zQV14=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10/go.mod h1:1U2nGytADgY4/U7OykO3LfzeS3/zz4zqg7wlUA7XHfc=
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
|
||||
|
|
@ -1060,6 +1065,8 @@ golang.org/x/net v0.0.0-20220420153159-1850ba15e1be/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
|||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -1093,8 +1100,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
|
||||
golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -1332,8 +1339,8 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR
|
|||
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
|
||||
google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
|
||||
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
|
||||
google.golang.org/api v0.81.0 h1:o8WF5AvfidafWbFjsRyupxyEQJNUWxLZJCK5NXrxZZ8=
|
||||
google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko=
|
||||
google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc=
|
||||
google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
|
@ -1436,8 +1443,8 @@ google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX
|
|||
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I=
|
||||
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
|
||||
google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM=
|
||||
google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
|
@ -1513,6 +1520,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
|||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.10"
|
||||
version: "3.10"
|
||||
appVersion: "3.11"
|
||||
version: "3.11"
|
||||
|
|
|
|||
|
|
@ -539,11 +539,12 @@
|
|||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{type}}",
|
||||
"legendFormat": "{{bucket}} {{type}}",
|
||||
"refId": "B",
|
||||
"step": 60
|
||||
}
|
||||
|
|
@ -645,11 +646,12 @@
|
|||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{type}}",
|
||||
"legendFormat": "{{bucket}} {{type}}",
|
||||
"refId": "B",
|
||||
"step": 60
|
||||
}
|
||||
|
|
@ -751,11 +753,11 @@
|
|||
"step": 60
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{type}}",
|
||||
"legendFormat": "{{bucket}} {{type}}",
|
||||
"refId": "B",
|
||||
"step": 60
|
||||
}
|
||||
|
|
@ -864,7 +866,7 @@
|
|||
"expr": "rate(SeaweedFS_s3_request_total[1m])",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{type}}",
|
||||
"legendFormat": "{{bucket}} {{type}}",
|
||||
"refId": "A",
|
||||
"step": 30
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var Commands = []*Command{
|
||||
|
|
@ -36,6 +37,7 @@ var Commands = []*Command{
|
|||
cmdScaffold,
|
||||
cmdServer,
|
||||
cmdShell,
|
||||
cmdUpdate,
|
||||
cmdUpload,
|
||||
cmdVersion,
|
||||
cmdVolume,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ type FilerOptions struct {
|
|||
debug *bool
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -82,6 +83,7 @@ func init() {
|
|||
f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
|
||||
f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
|
|
@ -216,6 +218,7 @@ func (fo *FilerOptions) startFiler() {
|
|||
Cipher: *fo.cipher,
|
||||
SaveToFilerLimit: int64(*fo.saveToFilerLimit),
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
|
||||
"github.com/chrislusf/seaweedfs/weed/replication/source"
|
||||
"github.com/chrislusf/seaweedfs/weed/security"
|
||||
statsCollect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"google.golang.org/grpc"
|
||||
|
|
@ -40,6 +41,7 @@ type SyncOptions struct {
|
|||
bFromTsMs *int64
|
||||
aProxyByFiler *bool
|
||||
bProxyByFiler *bool
|
||||
metricsHttpPort *int
|
||||
clientId int32
|
||||
}
|
||||
|
||||
|
|
@ -72,6 +74,7 @@ func init() {
|
|||
syncOptions.bFromTsMs = cmdFilerSynchronize.Flag.Int64("b.fromTsMs", 0, "synchronization from timestamp on filer B. The unit is millisecond")
|
||||
syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
|
||||
syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
|
||||
syncOptions.clientId = util.RandomInt32()
|
||||
}
|
||||
|
||||
|
|
@ -103,6 +106,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
|||
filerA := pb.ServerAddress(*syncOptions.filerA)
|
||||
filerB := pb.ServerAddress(*syncOptions.filerB)
|
||||
|
||||
// start filer.sync metrics server
|
||||
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort)
|
||||
|
||||
// read a filer signature
|
||||
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
|
||||
if aFilerErr != nil {
|
||||
|
|
@ -182,7 +188,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
|
|||
|
||||
// if first time, start from now
|
||||
// if has previously synced, resume from that point of time
|
||||
sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature)
|
||||
sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -210,14 +216,17 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
|
|||
}
|
||||
|
||||
var lastLogTsNs = time.Now().Nanosecond()
|
||||
var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
|
||||
processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
|
||||
now := time.Now().Nanosecond()
|
||||
glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
|
||||
lastLogTsNs = now
|
||||
return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
|
||||
// collect synchronous offset
|
||||
statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(lastTsNs))
|
||||
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs)
|
||||
})
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId,
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId,
|
||||
sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
|
||||
|
||||
}
|
||||
|
|
@ -226,6 +235,16 @@ const (
|
|||
SyncKeyPrefix = "sync."
|
||||
)
|
||||
|
||||
// When each business is distinguished according to path, and offsets need to be maintained separately.
|
||||
func getSignaturePrefixByPath(path string) string {
|
||||
// compatible historical version
|
||||
if path == "/" {
|
||||
return SyncKeyPrefix
|
||||
} else {
|
||||
return SyncKeyPrefix + path
|
||||
}
|
||||
}
|
||||
|
||||
func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
|
||||
|
||||
readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ package command
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
|
@ -12,7 +10,9 @@ import (
|
|||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/chrislusf/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
type ServerOptions struct {
|
||||
|
|
@ -114,6 +114,7 @@ func init() {
|
|||
filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
|
||||
filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
|
||||
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
|
||||
|
|
@ -131,6 +132,7 @@ func init() {
|
|||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
|
||||
serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, "<exprimental> enable tcp port")
|
||||
serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
|
||||
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
|
||||
|
|
|
|||
382
weed/command/update.go
Normal file
382
weed/command/update.go
Normal file
|
|
@ -0,0 +1,382 @@
|
|||
package command
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
)
|
||||
|
||||
//copied from https://github.com/restic/restic/tree/master/internal/selfupdate
|
||||
|
||||
// Release collects data about a single release on GitHub.
|
||||
type Release struct {
|
||||
Name string `json:"name"`
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
PreRelease bool `json:"prerelease"`
|
||||
PublishedAt time.Time `json:"published_at"`
|
||||
Assets []Asset `json:"assets"`
|
||||
|
||||
Version string `json:"-"` // set manually in the code
|
||||
}
|
||||
|
||||
// Asset is a file uploaded and attached to a release.
|
||||
type Asset struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
const githubAPITimeout = 30 * time.Second
|
||||
|
||||
// githubError is returned by the GitHub API, e.g. for rate-limiting.
|
||||
type githubError struct {
|
||||
Message string
|
||||
}
|
||||
|
||||
//default version is not full version
|
||||
var isFullVersion = false
|
||||
|
||||
var (
|
||||
updateOpt UpdateOptions
|
||||
)
|
||||
|
||||
type UpdateOptions struct {
|
||||
dir *string
|
||||
name *string
|
||||
Version *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
path, _ := os.Executable()
|
||||
_, name := filepath.Split(path)
|
||||
updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.")
|
||||
updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed. On windows, name shouldn't be same to the orignial name.")
|
||||
updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.")
|
||||
cmdUpdate.Run = runUpdate
|
||||
}
|
||||
|
||||
var cmdUpdate = &Command{
|
||||
UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]",
|
||||
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
|
||||
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
|
||||
}
|
||||
|
||||
func runUpdate(cmd *Command, args []string) bool {
|
||||
path, _ := os.Executable()
|
||||
_, name := filepath.Split(path)
|
||||
|
||||
if *updateOpt.dir != "" {
|
||||
if err := util.TestFolderWritable(util.ResolvePath(*updateOpt.dir)); err != nil {
|
||||
glog.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err)
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
*updateOpt.dir = filepath.Dir(path)
|
||||
}
|
||||
|
||||
if *updateOpt.name == "" {
|
||||
*updateOpt.name = name
|
||||
}
|
||||
|
||||
target := filepath.Join(*updateOpt.dir, *updateOpt.name)
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
if target == path {
|
||||
glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(0).Infof("new weed will be saved to %s", target)
|
||||
|
||||
_, err := downloadRelease(context.Background(), target, *updateOpt.Version)
|
||||
if err != nil {
|
||||
glog.Errorf("unable to download weed: %v", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
|
||||
currentVersion := util.VERSION_NUMBER
|
||||
rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if rel.Version == currentVersion {
|
||||
if ver == "0" {
|
||||
glog.V(0).Infof("weed is up to date")
|
||||
} else {
|
||||
glog.V(0).Infof("no need to download the same version of weed ")
|
||||
}
|
||||
return currentVersion, nil
|
||||
}
|
||||
|
||||
glog.V(0).Infof("download version: %s", rel.Version)
|
||||
|
||||
largeDiskSuffix := ""
|
||||
if util.VolumeSizeLimitGB == 8000 {
|
||||
largeDiskSuffix = "_large_disk"
|
||||
}
|
||||
|
||||
fullSuffix := ""
|
||||
if isFullVersion {
|
||||
fullSuffix = "_full"
|
||||
}
|
||||
|
||||
ext := "tar.gz"
|
||||
if runtime.GOOS == "windows" {
|
||||
ext = "zip"
|
||||
}
|
||||
|
||||
suffix := fmt.Sprintf("%s_%s%s%s.%s", runtime.GOOS, runtime.GOARCH, fullSuffix, largeDiskSuffix, ext)
|
||||
md5Filename := fmt.Sprintf("%s.md5", suffix)
|
||||
_, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
downloadFilename, buf, err := getGithubDataFile(ctx, rel.Assets, suffix)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
md5Ctx := md5.New()
|
||||
md5Ctx.Write(buf)
|
||||
binaryMd5 := md5Ctx.Sum(nil)
|
||||
if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) {
|
||||
glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32]))
|
||||
err = fmt.Errorf("binary md5sum doesn't match")
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = extractToFile(buf, downloadFilename, target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
} else {
|
||||
glog.V(0).Infof("successfully updated weed to version %v\n", rel.Version)
|
||||
}
|
||||
|
||||
return rel.Version, nil
|
||||
}
|
||||
|
||||
// GitHubLatestRelease uses the GitHub API to get information about the specific
|
||||
// release of a repository.
|
||||
func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (Release, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, githubAPITimeout)
|
||||
defer cancel()
|
||||
|
||||
url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
|
||||
// pin API version 3
|
||||
req.Header.Set("Accept", "application/vnd.github.v3+json")
|
||||
|
||||
res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
content := res.Header.Get("Content-Type")
|
||||
if strings.Contains(content, "application/json") {
|
||||
// try to decode error message
|
||||
var msg githubError
|
||||
jerr := json.NewDecoder(res.Body).Decode(&msg)
|
||||
if jerr == nil {
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message)
|
||||
}
|
||||
}
|
||||
|
||||
_ = res.Body.Close()
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
_ = res.Body.Close()
|
||||
return Release{}, err
|
||||
}
|
||||
|
||||
err = res.Body.Close()
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
|
||||
var release Release
|
||||
var releaseList []Release
|
||||
err = json.Unmarshal(buf, &releaseList)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
if ver == "0" {
|
||||
release = releaseList[0]
|
||||
glog.V(0).Infof("latest version is %v\n", release.TagName)
|
||||
} else {
|
||||
for _, r := range releaseList {
|
||||
if r.TagName == ver {
|
||||
release = r
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if release.TagName == "" {
|
||||
return Release{}, fmt.Errorf("can not find the specific version")
|
||||
}
|
||||
|
||||
release.Version = release.TagName
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func getGithubData(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// request binary data
|
||||
req.Header.Set("Accept", "application/octet-stream")
|
||||
|
||||
res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
_ = res.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = res.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func getGithubDataFile(ctx context.Context, assets []Asset, suffix string) (filename string, data []byte, err error) {
|
||||
var url string
|
||||
for _, a := range assets {
|
||||
if strings.HasSuffix(a.Name, suffix) {
|
||||
url = a.URL
|
||||
filename = a.Name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if url == "" {
|
||||
return "", nil, fmt.Errorf("unable to find file with suffix %v", suffix)
|
||||
}
|
||||
|
||||
glog.V(0).Infof("download %v\n", filename)
|
||||
data, err = getGithubData(ctx, url)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
return filename, data, nil
|
||||
}
|
||||
|
||||
func extractToFile(buf []byte, filename, target string) error {
|
||||
var rd io.Reader = bytes.NewReader(buf)
|
||||
|
||||
switch filepath.Ext(filename) {
|
||||
case ".gz":
|
||||
gr, err := gzip.NewReader(rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
trd := tar.NewReader(gr)
|
||||
hdr, terr := trd.Next()
|
||||
if terr != nil {
|
||||
glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
|
||||
return terr
|
||||
}
|
||||
rd = trd
|
||||
case ".zip":
|
||||
zrd, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(zrd.File) != 1 {
|
||||
return fmt.Errorf("ZIP archive contains more than one file")
|
||||
}
|
||||
|
||||
file, err := zrd.File[0].Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
rd = file
|
||||
}
|
||||
|
||||
// Write everything to a temp file
|
||||
dir := filepath.Dir(target)
|
||||
new, err := ioutil.TempFile(dir, "weed")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := io.Copy(new, rd)
|
||||
if err != nil {
|
||||
_ = new.Close()
|
||||
_ = os.Remove(new.Name())
|
||||
return err
|
||||
}
|
||||
if err = new.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = new.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mode := os.FileMode(0755)
|
||||
// attempt to find the original mode
|
||||
if fi, err := os.Lstat(target); err == nil {
|
||||
mode = fi.Mode()
|
||||
}
|
||||
|
||||
// Rename the temp file to the final location atomically.
|
||||
if err := os.Rename(new.Name(), target); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
glog.V(0).Infof("saved %d bytes in %v\n", n, target)
|
||||
return os.Chmod(target, mode)
|
||||
}
|
||||
9
weed/command/update_full.go
Normal file
9
weed/command/update_full.go
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
//go:build elastic && ydb && gocdk && hdfs
|
||||
// +build elastic,ydb,gocdk,hdfs
|
||||
|
||||
package command
|
||||
|
||||
//set true if gtags are set
|
||||
func init() {
|
||||
isFullVersion = true
|
||||
}
|
||||
|
|
@ -65,7 +65,8 @@ type VolumeServerOptions struct {
|
|||
preStopSeconds *int
|
||||
metricsHttpPort *int
|
||||
// pulseSeconds *int
|
||||
enableTcp *bool
|
||||
enableTcp *bool
|
||||
inflightUploadDataTimeout *time.Duration
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -96,6 +97,7 @@ func init() {
|
|||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
|
||||
v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<experimental> enable tcp port")
|
||||
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
}
|
||||
|
||||
var cmdVolume = &Command{
|
||||
|
|
@ -244,6 +246,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
|||
*v.fileSizeLimitMB,
|
||||
int64(*v.concurrentUploadLimitMB)*1024*1024,
|
||||
int64(*v.concurrentDownloadLimitMB)*1024*1024,
|
||||
*v.inflightUploadDataTimeout,
|
||||
)
|
||||
// starting grpc server
|
||||
grpcS := v.startGrpcService(volumeServer)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package filer
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
|
|
@ -11,6 +10,8 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
|
|
@ -63,14 +64,14 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
|
|||
|
||||
resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk)
|
||||
if err != nil {
|
||||
return chunks, nil, err
|
||||
return dataChunks, nil, err
|
||||
}
|
||||
|
||||
manifestChunks = append(manifestChunks, chunk)
|
||||
// recursive
|
||||
subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
|
||||
if subErr != nil {
|
||||
return chunks, nil, subErr
|
||||
return dataChunks, nil, subErr
|
||||
}
|
||||
dataChunks = append(dataChunks, subDataChunks...)
|
||||
manifestChunks = append(manifestChunks, subManifestChunks...)
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@ package filer
|
|||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/exp/slices"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
|
@ -248,6 +249,9 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n
|
|||
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
|
||||
|
||||
chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
visibles2 := readResolvedChunks(chunks)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package s3api
|
||||
|
||||
import (
|
||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
|
||||
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
|
@ -27,11 +28,12 @@ func (r *StatusRecorder) Flush() {
|
|||
|
||||
func track(f http.HandlerFunc, action string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
w.Header().Set("Server", "SeaweedFS S3")
|
||||
recorder := NewStatusResponseWriter(w)
|
||||
start := time.Now()
|
||||
f(recorder, r)
|
||||
stats_collect.S3RequestHistogram.WithLabelValues(action).Observe(time.Since(start).Seconds())
|
||||
stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status)).Inc()
|
||||
stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
|
||||
stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
|||
if rangeReq == "" {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
|
||||
if err := writeFn(bufferedWriter, 0, totalSize); err != nil {
|
||||
glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
|
@ -294,6 +295,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
|||
//mostly copy from src/pkg/net/http/fs.go
|
||||
ranges, err := parseRange(rangeReq, totalSize)
|
||||
if err != nil {
|
||||
glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
|
||||
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
|
|
@ -326,6 +328,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
|||
w.WriteHeader(http.StatusPartialContent)
|
||||
err = writeFn(bufferedWriter, ra.start, ra.length)
|
||||
if err != nil {
|
||||
glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
|
@ -365,6 +368,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
|
|||
}
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil {
|
||||
glog.Errorf("processRangeRequest err: %v", err)
|
||||
http.Error(w, "Internal Error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package weed_server
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/stats"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -229,6 +230,9 @@ func (fs *FilerServer) eachEventNotificationFn(req *filer_pb.SubscribeMetadataRe
|
|||
}
|
||||
}
|
||||
|
||||
// collect timestamps for path
|
||||
stats.FilerServerLastSendTsOfSubscribeGauge.WithLabelValues(fs.option.Host.String(), req.ClientName, req.PathPrefix).Set(float64(tsNs))
|
||||
|
||||
message := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: dirPath,
|
||||
EventNotification: eventNotification,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package weed_server
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
|
|
@ -17,6 +16,7 @@ import (
|
|||
|
||||
"github.com/chrislusf/seaweedfs/weed/operation"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
|
||||
|
|
@ -67,6 +67,7 @@ type FilerOption struct {
|
|||
Cipher bool
|
||||
SaveToFilerLimit int64
|
||||
ConcurrentUploadLimit int64
|
||||
ShowUIDirectoryDelete bool
|
||||
}
|
||||
|
||||
type FilerServer struct {
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
return
|
||||
}
|
||||
|
||||
ui.StatusTpl.Execute(w, struct {
|
||||
err = ui.StatusTpl.Execute(w, struct {
|
||||
Path string
|
||||
Breadcrumbs []ui.Breadcrumb
|
||||
Entries interface{}
|
||||
|
|
@ -81,6 +81,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
LastFileName string
|
||||
ShouldDisplayLoadMore bool
|
||||
EmptyFolder bool
|
||||
ShowDirectoryDelete bool
|
||||
}{
|
||||
path,
|
||||
ui.ToBreadcrumb(path),
|
||||
|
|
@ -89,5 +90,9 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
|||
lastFileName,
|
||||
shouldDisplayLoadMore,
|
||||
emptyFolder,
|
||||
fs.option.ShowUIDirectoryDelete,
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Template Execute Error: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,38 +109,37 @@
|
|||
<form class="upload-form">
|
||||
<input type="file" id="fileElem" multiple onchange="handleFiles(this.files)">
|
||||
|
||||
{{if .EmptyFolder}}
|
||||
{{ if .EmptyFolder }}
|
||||
<div class="row add-files">
|
||||
+
|
||||
</div>
|
||||
{{else}}
|
||||
{{ else }}
|
||||
<table width="100%" class="table table-hover">
|
||||
{{$path := .Path }}
|
||||
{{ $path := .Path }}
|
||||
{{ $showDirDel := .ShowDirectoryDelete }}
|
||||
{{ range $entry_index, $entry := .Entries }}
|
||||
<tr>
|
||||
<td>
|
||||
{{if $entry.IsDirectory}}
|
||||
{{ if $entry.IsDirectory }}
|
||||
<span class="glyphicon glyphicon-folder-open" aria-hidden="true"></span>
|
||||
<a href="{{ printpath $path "/" $entry.Name "/"}}" >
|
||||
{{ $entry.Name }}
|
||||
</a>
|
||||
{{else}}
|
||||
{{ else }}
|
||||
<a href="{{ printpath $path "/" $entry.Name }}" >
|
||||
{{ $entry.Name }}
|
||||
</a>
|
||||
{{end}}
|
||||
{{ end }}
|
||||
</td>
|
||||
<td align="right" nowrap>
|
||||
{{if $entry.IsDirectory}}
|
||||
{{else}}
|
||||
{{ if not $entry.IsDirectory }}
|
||||
{{ $entry.Mime }}
|
||||
{{end}}
|
||||
{{ end }}
|
||||
</td>
|
||||
<td align="right" nowrap>
|
||||
{{if $entry.IsDirectory}}
|
||||
{{else}}
|
||||
{{ if not $entry.IsDirectory }}
|
||||
{{ $entry.Size | humanizeBytes }}
|
||||
{{end}}
|
||||
{{ end }}
|
||||
</td>
|
||||
<td align="right" nowrap>
|
||||
{{ $entry.Timestamp.Format "2006-01-02 15:04" }}
|
||||
|
|
@ -150,31 +149,32 @@
|
|||
<label class="btn" onclick="handleRename('{{ $entry.Name }}', '{{ printpath $path "/" }}')">
|
||||
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{if $entry.IsDirectory}}
|
||||
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
|
||||
{{ if and $entry.IsDirectory $showDirDel }}
|
||||
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
|
||||
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{else}}
|
||||
{{ end }}
|
||||
{{ if not $entry.IsDirectory }}
|
||||
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name }}')">
|
||||
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
|
||||
</label>
|
||||
{{end}}
|
||||
{{ end }}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
</table>
|
||||
{{end}}
|
||||
{{ end }}
|
||||
</form>
|
||||
</div>
|
||||
|
||||
{{if .ShouldDisplayLoadMore}}
|
||||
{{ if .ShouldDisplayLoadMore }}
|
||||
<div class="row">
|
||||
<a href={{ print .Path "?limit=" .Limit "&lastFileName=" .LastFileName}} >
|
||||
<a href={{ print .Path "?limit=" .Limit "&lastFileName=" .LastFileName }} >
|
||||
Load more
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
{{ end }}
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package weed_server
|
|||
import (
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
|
||||
|
|
@ -24,7 +25,9 @@ type VolumeServer struct {
|
|||
inFlightDownloadDataSize int64
|
||||
concurrentUploadLimit int64
|
||||
concurrentDownloadLimit int64
|
||||
inFlightUploadDataLimitCond *sync.Cond
|
||||
inFlightDownloadDataLimitCond *sync.Cond
|
||||
inflightUploadDataTimeout time.Duration
|
||||
|
||||
SeedMasterNodes []pb.ServerAddress
|
||||
currentMaster pb.ServerAddress
|
||||
|
|
@ -60,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
|||
fileSizeLimitMB int,
|
||||
concurrentUploadLimit int64,
|
||||
concurrentDownloadLimit int64,
|
||||
inflightUploadDataTimeout time.Duration,
|
||||
) *VolumeServer {
|
||||
|
||||
v := util.GetViper()
|
||||
|
|
@ -84,9 +88,11 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
|
|||
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
|
||||
isHeartbeating: true,
|
||||
stopChan: make(chan bool),
|
||||
inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
|
||||
concurrentUploadLimit: concurrentUploadLimit,
|
||||
concurrentDownloadLimit: concurrentDownloadLimit,
|
||||
inflightUploadDataTimeout: inflightUploadDataTimeout,
|
||||
}
|
||||
vs.SeedMasterNodes = masterNodes
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
|
||||
|
|
@ -56,20 +57,31 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
|
|||
vs.guard.WhiteList(vs.DeleteHandler)(w, r)
|
||||
case "PUT", "POST":
|
||||
|
||||
// wait until in flight data is less than the limit
|
||||
contentLength := getContentLength(r)
|
||||
|
||||
// exclude the replication from the concurrentUploadLimitMB
|
||||
if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" &&
|
||||
atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
|
||||
err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
|
||||
glog.V(1).Infof("too many requests: %v", err)
|
||||
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
||||
return
|
||||
if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
|
||||
startTime := time.Now()
|
||||
vs.inFlightUploadDataLimitCond.L.Lock()
|
||||
for vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
|
||||
//wait timeout check
|
||||
if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
|
||||
vs.inFlightUploadDataLimitCond.L.Unlock()
|
||||
err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
|
||||
glog.V(1).Infof("too many requests: %v", err)
|
||||
writeJsonError(w, r, http.StatusTooManyRequests, err)
|
||||
return
|
||||
}
|
||||
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
|
||||
vs.inFlightUploadDataLimitCond.Wait()
|
||||
}
|
||||
vs.inFlightUploadDataLimitCond.L.Unlock()
|
||||
}
|
||||
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
|
||||
defer func() {
|
||||
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
|
||||
if vs.concurrentUploadLimit != 0 {
|
||||
vs.inFlightUploadDataLimitCond.Signal()
|
||||
}
|
||||
}()
|
||||
|
||||
// processs uploads
|
||||
|
|
|
|||
|
|
@ -77,6 +77,14 @@ var (
|
|||
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
||||
}, []string{"type"})
|
||||
|
||||
FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "SeaweedFS",
|
||||
Subsystem: "filer",
|
||||
Name: "last_send_timestamp_of_subscribe",
|
||||
Help: "The last send timestamp of the filer subscription.",
|
||||
}, []string{"sourceFiler", "clientName", "path"})
|
||||
|
||||
FilerStoreCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "SeaweedFS",
|
||||
|
|
@ -94,6 +102,14 @@ var (
|
|||
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
||||
}, []string{"store", "type"})
|
||||
|
||||
FilerSyncOffsetGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "SeaweedFS",
|
||||
Subsystem: "filerSync",
|
||||
Name: "sync_offset",
|
||||
Help: "The offset of the filer synchronization service.",
|
||||
}, []string{"sourceFiler", "targetFiler", "clientName", "path"})
|
||||
|
||||
VolumeServerRequestCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "SeaweedFS",
|
||||
|
|
@ -157,7 +173,8 @@ var (
|
|||
Subsystem: "s3",
|
||||
Name: "request_total",
|
||||
Help: "Counter of s3 requests.",
|
||||
}, []string{"type", "code"})
|
||||
}, []string{"type", "code", "bucket"})
|
||||
|
||||
S3RequestHistogram = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: "SeaweedFS",
|
||||
|
|
@ -165,7 +182,7 @@ var (
|
|||
Name: "request_seconds",
|
||||
Help: "Bucketed histogram of s3 request processing time.",
|
||||
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
|
||||
}, []string{"type"})
|
||||
}, []string{"type", "bucket"})
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -179,6 +196,8 @@ func init() {
|
|||
Gather.MustRegister(FilerRequestHistogram)
|
||||
Gather.MustRegister(FilerStoreCounter)
|
||||
Gather.MustRegister(FilerStoreHistogram)
|
||||
Gather.MustRegister(FilerSyncOffsetGauge)
|
||||
Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
|
||||
Gather.MustRegister(collectors.NewGoCollector())
|
||||
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ func (n *Needle) ReadNeedleDataInto(r backend.BackendStorageFile, volumeOffset i
|
|||
crc := CRC(0)
|
||||
for x := needleOffset; x < needleOffset+size; x += int64(len(buf)) {
|
||||
count, err := n.ReadNeedleData(r, volumeOffset, buf, x)
|
||||
if count > 0 {
|
||||
crc = crc.Update(buf[0:count])
|
||||
if _, err = writer.Write(buf[0:count]); err != nil {
|
||||
toWrite := min(int64(count), needleOffset+size-x)
|
||||
if toWrite > 0 {
|
||||
crc = crc.Update(buf[0:toWrite])
|
||||
if _, err = writer.Write(buf[0:toWrite]); err != nil {
|
||||
return fmt.Errorf("ReadNeedleData write: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
batch = 100000
|
||||
batch = 10000
|
||||
)
|
||||
|
||||
type SectionalNeedleId uint32
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.10)
|
||||
VERSION_NUMBER = fmt.Sprintf("%.02f", 3.11)
|
||||
VERSION = sizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
)
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) {
|
|||
return
|
||||
}
|
||||
locations, found = vc.ecVid2Locations[vid]
|
||||
return
|
||||
return locations, found && len(locations) > 0
|
||||
}
|
||||
|
||||
func (vc *vidMap) addLocation(vid uint32, location Location) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue