diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 876acd7cf..f2d057b90 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -2,12 +2,13 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net/http" "os" "strings" "sync" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -31,6 +32,8 @@ type IdentityAccessManagement struct { identities []*Identity isAuthEnabled bool domain string + hashes map[string]*sync.Pool + hashMu sync.RWMutex } type Identity struct { @@ -77,6 +80,7 @@ func (action Action) getPermission() Permission { func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ domain: option.DomainName, + hashes: make(map[string]*sync.Pool), } if option.Config != "" { if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 02a6bd4e0..06cdf67e4 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -23,6 +23,7 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" + "hash" "io" "net/http" "net/url" @@ -30,6 +31,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "unicode/utf8" @@ -151,14 +153,14 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope()) - // Get hmac signing key. - signingKey := getSigningKey(cred.SecretKey, + // Calculate signature. + newSignature := iam.getSignature( + cred.SecretKey, signV4Values.Credential.scope.date, signV4Values.Credential.scope.region, - signV4Values.Credential.scope.service) - - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) + signV4Values.Credential.scope.service, + stringToSign, + ) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { @@ -325,11 +327,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http. return s3err.ErrInvalidAccessKeyID } - // Get signing key. - signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, credHeader.scope.service) - // Get signature. - newSignature := getSignature(signingKey, formValues.Get("Policy")) + newSignature := iam.getSignature( + cred.SecretKey, + credHeader.scope.date, + credHeader.scope.region, + credHeader.scope.service, + formValues.Get("Policy"), + ) // Verify signature. if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { @@ -442,14 +447,14 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s // Get string to sign from canonical request. presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope()) - // Get hmac presigned signing key. - presignedSigningKey := getSigningKey(cred.SecretKey, + // Get new signature. + newSignature := iam.getSignature( + cred.SecretKey, pSignValues.Credential.scope.date, pSignValues.Credential.scope.region, - pSignValues.Credential.scope.service) - - // Get new signature. - newSignature := getSignature(presignedSigningKey, presignedStringToSign) + pSignValues.Credential.scope.service, + presignedStringToSign, + ) // Verify signature. if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) { @@ -458,6 +463,38 @@ func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload s return identity, s3err.ErrNone } +// getSignature +func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string { + date := t.Format(yyyymmdd) + hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request" + + iam.hashMu.RLock() + pool, ok := iam.hashes[hashID] + iam.hashMu.RUnlock() + + if !ok { + iam.hashMu.Lock() + if pool, ok = iam.hashes[hashID]; !ok { + pool = &sync.Pool{ + New: func() any { + signingKey := getSigningKey(secretKey, date, region, service) + return hmac.New(sha256.New, signingKey) + }, + } + iam.hashes[hashID] = pool + } + iam.hashMu.Unlock() + } + + h := pool.Get().(hash.Hash) + h.Reset() + h.Write([]byte(stringToSign)) + sig := hex.EncodeToString(h.Sum(nil)) + pool.Put(h) + + return sig +} + func contains(list []string, elem string) bool { for _, t := range list { if t == elem { @@ -674,19 +711,14 @@ func sumHMAC(key []byte, data []byte) []byte { } // getSigningKey hmac seed to calculate final signature. -func getSigningKey(secretKey string, t time.Time, region string, service string) []byte { - date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd))) +func getSigningKey(secretKey string, time string, region string, service string) []byte { + date := sumHMAC([]byte("AWS4"+secretKey), []byte(time)) regionBytes := sumHMAC(date, []byte(region)) serviceBytes := sumHMAC(regionBytes, []byte(service)) signingKey := sumHMAC(serviceBytes, []byte("aws4_request")) return signingKey } -// getSignature final signature in hexadecimal form. -func getSignature(signingKey []byte, stringToSign string) string { - return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) -} - // getCanonicalHeaders generate a list of request headers with their values func getCanonicalHeaders(signedHeaders http.Header) string { var headers []string diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index db8bfd8ef..15ca90b93 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -14,6 +14,7 @@ import ( "sort" "strconv" "strings" + "sync" "testing" "time" "unicode/utf8" @@ -114,7 +115,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) { }{ {Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrAccessDenied}, {Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, - {Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, + {Request: mustNewPresignedRequest(iam, "GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, } for i, testCase := range testCases { if _, s3Error := iam.reqSignatureV4Verify(testCase.Request); s3Error != testCase.ErrCode { @@ -123,6 +124,19 @@ func TestCheckAdminRequestAuthType(t *testing.T) { } } +func BenchmarkGetSignature(b *testing.B) { + t := time.Now() + iam := IdentityAccessManagement{ + hashes: make(map[string]*sync.Pool), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + iam.getSignature("secret-key", t, "us-east-1", "s3", "random data") + } +} + // Provides a fully populated http request instance, fails otherwise. func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { req, err := newTestRequest(method, urlStr, contentLength, body) @@ -145,10 +159,10 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod // This is similar to mustNewRequest but additionally the request // is presigned with AWS Signature V4, fails if not able to do so. -func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { +func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { req := mustNewRequest(method, urlStr, contentLength, body, t) cred := &Credential{"access_key_1", "secret_key_1"} - if err := preSignV4(req, cred.AccessKey, cred.SecretKey, int64(10*time.Minute.Seconds())); err != nil { + if err := preSignV4(iam, req, cred.AccessKey, cred.SecretKey, int64(10*time.Minute.Seconds())); err != nil { t.Fatalf("Unable to initialized new signed http request %s", err) } return req @@ -343,7 +357,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error { // preSignV4 presign the request, in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. -func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { +func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return errors.New("Presign cannot be generated without access and secret keys") @@ -370,8 +384,7 @@ func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires i queryStr := strings.Replace(query.Encode(), "+", "%20", -1) canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method) stringToSign := getStringToSign(canonicalRequest, date, scope) - signingKey := getSigningKey(secretAccessKey, date, region, "s3") - signature := getSignature(signingKey, stringToSign) + signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign) req.URL.RawQuery = query.Encode() diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index 8ba1bc479..4bf74d025 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -24,36 +24,17 @@ import ( "crypto/sha256" "encoding/hex" "errors" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "hash" "io" "net/http" "time" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/dustin/go-humanize" ) -// getChunkSignature - get chunk signature. -func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string { - - // Calculate string to sign. - stringToSign := signV4ChunkedAlgorithm + "\n" + - date.Format(iso8601Format) + "\n" + - getScope(date, region) + "\n" + - seedSignature + "\n" + - emptySHA256 + "\n" + - hashedChunk - - // Get hmac signing key. - signingKey := getSigningKey(secretKey, date, region, "s3") - - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) - - return newSignature -} - // calculateSeedSignature - Calculate seed signature in accordance with // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html // @@ -124,11 +105,14 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) - // Get hmac signing key. - signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3") - // Calculate signature. - newSignature := getSignature(signingKey, stringToSign) + newSignature := iam.getSignature( + cred.SecretKey, + signV4Values.Credential.scope.date, + region, + "s3", + stringToSign, + ) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { @@ -163,6 +147,7 @@ func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) ( region: region, chunkSHA256Writer: sha256.New(), state: readChunkHeader, + iam: iam, }, s3err.ErrNone } @@ -180,6 +165,7 @@ type s3ChunkedReader struct { chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data. n uint64 // Unread bytes in chunk err error + iam *IdentityAccessManagement } // Read chunk reads the chunk token signature portion. @@ -296,7 +282,7 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { // Calculate the hashed chunk. hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) // Calculate the chunk signature. - newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk) + newSignature := cr.getChunkSignature(hashedChunk) if !compareSignatureV4(cr.chunkSignature, newSignature) { // Chunk signature doesn't match we return signature does not match. cr.err = errors.New("chunk signature does not match") @@ -317,6 +303,26 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { } } +// getChunkSignature - get chunk signature. +func (cr *s3ChunkedReader) getChunkSignature(hashedChunk string) string { + // Calculate string to sign. + stringToSign := signV4ChunkedAlgorithm + "\n" + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + emptySHA256 + "\n" + + hashedChunk + + // Calculate signature. + return cr.iam.getSignature( + cr.cred.SecretKey, + cr.seedDate, + cr.region, + "s3", + stringToSign, + ) +} + // readCRLF - check if reader only has '\r\n' CRLF character. // returns malformed encoding if it doesn't. func readCRLF(reader io.Reader) error {