mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge branch 'master' of https://github.com/chrislusf/seaweedfs
This commit is contained in:
commit
4076d74813
60
.github/workflows/binaries_release4.yml
vendored
Normal file
60
.github/workflows/binaries_release4.yml
vendored
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
# This is a basic workflow to help you get started with Actions
|
||||||
|
|
||||||
|
name: "go: build versioned binaries for linux with all tags"
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
|
||||||
|
# Allows you to run this workflow manually from the Actions tab
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
|
||||||
|
build-release-binaries_linux:
|
||||||
|
permissions:
|
||||||
|
contents: write # for wangyoucao577/go-release-action to upload release assets
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
goos: [linux]
|
||||||
|
goarch: [amd64]
|
||||||
|
|
||||||
|
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||||
|
steps:
|
||||||
|
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||||
|
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||||
|
- name: Go Release Binaries Normal Volume Size
|
||||||
|
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||||
|
with:
|
||||||
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
goos: ${{ matrix.goos }}
|
||||||
|
goarch: ${{ matrix.goarch }}
|
||||||
|
overwrite: true
|
||||||
|
build_flags: -tags elastic,ydb,gocdk,hdfs,rocksdb
|
||||||
|
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||||
|
# build_flags: -tags 5BytesOffset # optional, default is
|
||||||
|
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||||
|
# Where to run `go build .`
|
||||||
|
project_path: weed
|
||||||
|
binary_name: weed
|
||||||
|
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||||
|
- name: Go Release Large Disk Binaries
|
||||||
|
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
|
||||||
|
with:
|
||||||
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
goos: ${{ matrix.goos }}
|
||||||
|
goarch: ${{ matrix.goarch }}
|
||||||
|
overwrite: true
|
||||||
|
pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
|
||||||
|
build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,hdfs,rocksdb
|
||||||
|
ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
|
||||||
|
# Where to run `go build .`
|
||||||
|
project_path: weed
|
||||||
|
binary_name: weed
|
||||||
|
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full_large_disk"
|
43
.github/workflows/codeql.yml
vendored
Normal file
43
.github/workflows/codeql.yml
vendored
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
name: "Code Scanning - Action"
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
CodeQL-Build:
|
||||||
|
# CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
# required for all workflows
|
||||||
|
security-events: write
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
# Initializes the CodeQL tools for scanning.
|
||||||
|
- name: Initialize CodeQL
|
||||||
|
uses: github/codeql-action/init@v2
|
||||||
|
# Override language selection by uncommenting this and choosing your languages
|
||||||
|
with:
|
||||||
|
languages: go
|
||||||
|
|
||||||
|
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||||
|
# If this step fails, then you should remove it and run the build manually (see below).
|
||||||
|
- name: Autobuild
|
||||||
|
uses: github/codeql-action/autobuild@v2
|
||||||
|
|
||||||
|
# ℹ️ Command-line programs to run using the OS shell.
|
||||||
|
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||||
|
|
||||||
|
# ✏️ If the Autobuild fails above, remove it and uncomment the following
|
||||||
|
# three lines and modify them (or add more) to build your code if your
|
||||||
|
# project uses a compiled language
|
||||||
|
|
||||||
|
#- run: |
|
||||||
|
# make bootstrap
|
||||||
|
# make release
|
||||||
|
|
||||||
|
- name: Perform CodeQL Analysis
|
||||||
|
uses: github/codeql-action/analyze@v2
|
3
.github/workflows/container_release2.yml
vendored
3
.github/workflows/container_release2.yml
vendored
|
@ -52,7 +52,8 @@ jobs:
|
||||||
with:
|
with:
|
||||||
context: ./docker
|
context: ./docker
|
||||||
push: ${{ github.event_name != 'pull_request' }}
|
push: ${{ github.event_name != 'pull_request' }}
|
||||||
file: ./docker/Dockerfile.go_build_large
|
file: ./docker/Dockerfile.go_build
|
||||||
|
build-args: TAGS=5BytesOffset
|
||||||
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
platforms: linux/amd64, linux/arm, linux/arm64, linux/386
|
||||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||||
|
|
58
.github/workflows/container_release4.yml
vendored
Normal file
58
.github/workflows/container_release4.yml
vendored
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
name: "docker: build release containers for all tags"
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
workflow_dispatch: []
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-default-release-container:
|
||||||
|
runs-on: [ubuntu-latest]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
-
|
||||||
|
name: Checkout
|
||||||
|
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
||||||
|
-
|
||||||
|
name: Docker meta
|
||||||
|
id: docker_meta
|
||||||
|
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a # v3
|
||||||
|
with:
|
||||||
|
images: |
|
||||||
|
chrislusf/seaweedfs
|
||||||
|
tags: |
|
||||||
|
type=ref,event=tag,suffix=_full
|
||||||
|
flavor: |
|
||||||
|
latest=false
|
||||||
|
labels: |
|
||||||
|
org.opencontainers.image.title=seaweedfs
|
||||||
|
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||||
|
org.opencontainers.image.vendor=Chris Lu
|
||||||
|
-
|
||||||
|
name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # v1
|
||||||
|
-
|
||||||
|
name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1
|
||||||
|
-
|
||||||
|
name: Login to Docker Hub
|
||||||
|
if: github.event_name != 'pull_request'
|
||||||
|
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b # v1
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
-
|
||||||
|
name: Build
|
||||||
|
uses: docker/build-push-action@e551b19e49efd4e98792db7592c17c09b89db8d8 # v2
|
||||||
|
with:
|
||||||
|
context: ./docker
|
||||||
|
push: ${{ github.event_name != 'pull_request' }}
|
||||||
|
file: ./docker/Dockerfile.go_build
|
||||||
|
build-args: TAGS=elastic,ydb,gocdk,hdfs
|
||||||
|
platforms: linux/amd64
|
||||||
|
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.docker_meta.outputs.labels }}
|
|
@ -3,10 +3,11 @@ RUN apk add git g++ fuse
|
||||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
RUN mkdir -p /go/src/github.com/chrislusf/
|
||||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
||||||
ARG BRANCH=${BRANCH:-master}
|
ARG BRANCH=${BRANCH:-master}
|
||||||
|
ARG TAGS
|
||||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
||||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
||||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
||||||
&& CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}"
|
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||||
|
|
||||||
FROM alpine AS final
|
FROM alpine AS final
|
||||||
LABEL author="Chris Lu"
|
LABEL author="Chris Lu"
|
||||||
|
|
|
@ -1,43 +0,0 @@
|
||||||
FROM golang:1.18-alpine as builder
|
|
||||||
RUN apk add git g++ fuse
|
|
||||||
RUN mkdir -p /go/src/github.com/chrislusf/
|
|
||||||
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
|
|
||||||
ARG BRANCH=${BRANCH:-master}
|
|
||||||
RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
|
|
||||||
RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
|
|
||||||
&& export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
|
|
||||||
&& CGO_ENABLED=0 go install -tags 5BytesOffset -ldflags "-extldflags -static ${LDFLAGS}"
|
|
||||||
|
|
||||||
FROM alpine AS final
|
|
||||||
LABEL author="Chris Lu"
|
|
||||||
COPY --from=builder /go/bin/weed /usr/bin/
|
|
||||||
RUN mkdir -p /etc/seaweedfs
|
|
||||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
|
||||||
COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
|
||||||
RUN apk add fuse # for weed mount
|
|
||||||
|
|
||||||
# volume server gprc port
|
|
||||||
EXPOSE 18080
|
|
||||||
# volume server http port
|
|
||||||
EXPOSE 8080
|
|
||||||
# filer server gprc port
|
|
||||||
EXPOSE 18888
|
|
||||||
# filer server http port
|
|
||||||
EXPOSE 8888
|
|
||||||
# master server shared gprc port
|
|
||||||
EXPOSE 19333
|
|
||||||
# master server shared http port
|
|
||||||
EXPOSE 9333
|
|
||||||
# s3 server http port
|
|
||||||
EXPOSE 8333
|
|
||||||
# webdav server http port
|
|
||||||
EXPOSE 7333
|
|
||||||
|
|
||||||
RUN mkdir -p /data/filerldb2
|
|
||||||
|
|
||||||
VOLUME /data
|
|
||||||
WORKDIR /data
|
|
||||||
|
|
||||||
RUN chmod +x /entrypoint.sh
|
|
||||||
|
|
||||||
ENTRYPOINT ["/entrypoint.sh"]
|
|
|
@ -3,7 +3,7 @@ FROM golang:1.18-buster as builder
|
||||||
RUN apt-get update
|
RUN apt-get update
|
||||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||||
|
|
||||||
ENV ROCKSDB_VERSION v7.0.4
|
ENV ROCKSDB_VERSION v7.2.2
|
||||||
|
|
||||||
# build RocksDB
|
# build RocksDB
|
||||||
RUN cd /tmp && \
|
RUN cd /tmp && \
|
||||||
|
|
|
@ -7,14 +7,17 @@ gen: dev
|
||||||
binary:
|
binary:
|
||||||
export SWCOMMIT=$(shell git rev-parse --short HEAD)
|
export SWCOMMIT=$(shell git rev-parse --short HEAD)
|
||||||
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
export SWLDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
|
||||||
cd ../weed; CGO_ENABLED=0 GOOS=linux go build --tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
|
cd ../weed; CGO_ENABLED=0 GOOS=linux go build -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)"; mv weed ../docker/
|
||||||
|
|
||||||
build: binary
|
build: binary
|
||||||
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
|
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
|
||||||
rm ./weed
|
rm ./weed
|
||||||
|
|
||||||
build_gorocksdb:
|
go_build: # make go_build tags=elastic,ydb,gocdk,hdfs,rocksdb,5BytesOffset
|
||||||
docker build --no-cache -t chrislusf/gorocksdb -f Dockerfile.go_rocksdb .
|
docker build --build-arg TAGS=$(tags) --no-cache -t chrislusf/seaweedfs:go_build -f Dockerfile.go_build .
|
||||||
|
|
||||||
|
go_build_large_disk:
|
||||||
|
docker build --build-arg TAGS=large_disk --no-cache -t chrislusf/seaweedfs:large_disk -f Dockerfile.go_build .
|
||||||
|
|
||||||
build_rocksdb:
|
build_rocksdb:
|
||||||
docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
|
docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
|
||||||
|
|
|
@ -28,11 +28,14 @@ const (
|
||||||
AmzStorageClass = "x-amz-storage-class"
|
AmzStorageClass = "x-amz-storage-class"
|
||||||
|
|
||||||
// S3 user-defined metadata
|
// S3 user-defined metadata
|
||||||
AmzUserMetaPrefix = "X-Amz-Meta-"
|
AmzUserMetaPrefix = "X-Amz-Meta-"
|
||||||
|
AmzUserMetaDirective = "X-Amz-Metadata-Directive"
|
||||||
|
|
||||||
// S3 object tagging
|
// S3 object tagging
|
||||||
AmzObjectTagging = "X-Amz-Tagging"
|
AmzObjectTagging = "X-Amz-Tagging"
|
||||||
AmzTagCount = "x-amz-tagging-count"
|
AmzObjectTaggingPrefix = "X-Amz-Tagging-"
|
||||||
|
AmzObjectTaggingDirective = "X-Amz-Tagging-Directive"
|
||||||
|
AmzTagCount = "x-amz-tagging-count"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Non-Standard S3 HTTP request constants
|
// Non-Standard S3 HTTP request constants
|
||||||
|
|
|
@ -3,9 +3,10 @@ package s3api
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||||
|
headers "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||||
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||||
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
|
||||||
weed_server "github.com/chrislusf/seaweedfs/weed/server"
|
"modernc.org/strutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -15,6 +16,11 @@ import (
|
||||||
"github.com/chrislusf/seaweedfs/weed/util"
|
"github.com/chrislusf/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
DirectiveCopy = "COPY"
|
||||||
|
DirectiveReplace = "REPLACE"
|
||||||
|
)
|
||||||
|
|
||||||
func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
dstBucket, dstObject := xhttp.GetBucketAndObject(r)
|
dstBucket, dstObject := xhttp.GetBucketAndObject(r)
|
||||||
|
@ -30,7 +36,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||||
|
|
||||||
glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
|
glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
|
||||||
|
|
||||||
if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && isReplace(r) {
|
replaceMeta, replaceTagging := replaceDirective(r.Header)
|
||||||
|
|
||||||
|
if (srcBucket == dstBucket && srcObject == dstObject || cpSrcPath == "") && (replaceMeta || replaceTagging) {
|
||||||
fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
|
fullPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
|
||||||
dir, name := fullPath.DirAndName()
|
dir, name := fullPath.DirAndName()
|
||||||
entry, err := s3a.getEntry(dir, name)
|
entry, err := s3a.getEntry(dir, name)
|
||||||
|
@ -38,7 +46,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
|
entry.Extended = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging)
|
||||||
err = s3a.touch(dir, name, entry)
|
err = s3a.touch(dir, name, entry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||||
|
@ -80,6 +88,11 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||||
}
|
}
|
||||||
defer util.CloseResponse(resp)
|
defer util.CloseResponse(resp)
|
||||||
|
|
||||||
|
tagErr := processMetadata(r.Header, resp.Header, replaceMeta, replaceTagging, s3a.getTags, dir, name)
|
||||||
|
if tagErr != nil {
|
||||||
|
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||||
|
return
|
||||||
|
}
|
||||||
glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
|
glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl)
|
||||||
etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
|
etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
|
||||||
|
|
||||||
|
@ -182,6 +195,107 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isReplace(r *http.Request) bool {
|
func replaceDirective(reqHeader http.Header) (replaceMeta, replaceTagging bool) {
|
||||||
return r.Header.Get("X-Amz-Metadata-Directive") == "REPLACE"
|
return reqHeader.Get(headers.AmzUserMetaDirective) == DirectiveReplace, reqHeader.Get(headers.AmzObjectTaggingDirective) == DirectiveReplace
|
||||||
|
}
|
||||||
|
|
||||||
|
func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTagging bool, getTags func(parentDirectoryPath string, entryName string) (tags map[string]string, err error), dir, name string) (err error) {
|
||||||
|
if sc := reqHeader.Get(xhttp.AmzStorageClass); len(sc) == 0 {
|
||||||
|
if sc := existing[xhttp.AmzStorageClass]; len(sc) > 0 {
|
||||||
|
reqHeader[xhttp.AmzStorageClass] = sc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !replaceMeta {
|
||||||
|
for header, _ := range reqHeader {
|
||||||
|
if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
|
||||||
|
delete(reqHeader, header)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k, v := range existing {
|
||||||
|
if strings.HasPrefix(k, xhttp.AmzUserMetaPrefix) {
|
||||||
|
reqHeader[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !replaceTagging {
|
||||||
|
for header, _ := range reqHeader {
|
||||||
|
if strings.HasPrefix(header, xhttp.AmzObjectTagging) {
|
||||||
|
delete(reqHeader, header)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
found := false
|
||||||
|
for k, _ := range existing {
|
||||||
|
if strings.HasPrefix(k, xhttp.AmzObjectTaggingPrefix) {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if found {
|
||||||
|
tags, err := getTags(dir, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var tagArr []string
|
||||||
|
for k, v := range tags {
|
||||||
|
tagArr = append(tagArr, fmt.Sprintf("%s=%s", k, v))
|
||||||
|
}
|
||||||
|
tagStr := strutil.JoinFields(tagArr, "&")
|
||||||
|
reqHeader.Set(xhttp.AmzObjectTagging, tagStr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte) {
|
||||||
|
metadata = make(map[string][]byte)
|
||||||
|
|
||||||
|
if sc := existing[xhttp.AmzStorageClass]; len(sc) > 0 {
|
||||||
|
metadata[xhttp.AmzStorageClass] = sc
|
||||||
|
}
|
||||||
|
if sc := reqHeader.Get(xhttp.AmzStorageClass); len(sc) > 0 {
|
||||||
|
metadata[xhttp.AmzStorageClass] = []byte(sc)
|
||||||
|
}
|
||||||
|
|
||||||
|
if replaceMeta {
|
||||||
|
for header, values := range reqHeader {
|
||||||
|
if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) {
|
||||||
|
for _, value := range values {
|
||||||
|
metadata[header] = []byte(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for k, v := range existing {
|
||||||
|
if strings.HasPrefix(k, xhttp.AmzUserMetaPrefix) {
|
||||||
|
metadata[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if replaceTagging {
|
||||||
|
if tags := reqHeader.Get(xhttp.AmzObjectTagging); tags != "" {
|
||||||
|
for _, v := range strings.Split(tags, "&") {
|
||||||
|
tag := strings.Split(v, "=")
|
||||||
|
if len(tag) == 2 {
|
||||||
|
metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
|
||||||
|
} else if len(tag) == 1 {
|
||||||
|
metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for k, v := range existing {
|
||||||
|
if strings.HasPrefix(k, xhttp.AmzObjectTagging) {
|
||||||
|
metadata[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete(metadata, xhttp.AmzTagCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
426
weed/s3api/s3api_object_copy_handlers_test.go
Normal file
426
weed/s3api/s3api_object_copy_handlers_test.go
Normal file
|
@ -0,0 +1,426 @@
|
||||||
|
package s3api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
headers "github.com/chrislusf/seaweedfs/weed/s3api/http"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type H map[string]string
|
||||||
|
|
||||||
|
func (h H) String() string {
|
||||||
|
pairs := make([]string, 0, len(h))
|
||||||
|
for k, v := range h {
|
||||||
|
pairs = append(pairs, fmt.Sprintf("%s : %s", k, v))
|
||||||
|
}
|
||||||
|
sort.Strings(pairs)
|
||||||
|
join := strings.Join(pairs, "\n")
|
||||||
|
return "\n" + join + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
var processMetadataTestCases = []struct {
|
||||||
|
caseId int
|
||||||
|
request H
|
||||||
|
existing H
|
||||||
|
getTags H
|
||||||
|
want H
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
201,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=existing",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
202,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=existing",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
203,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
204,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
205,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{},
|
||||||
|
H{},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
206,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
207,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-Type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"A": "B",
|
||||||
|
"a": "b",
|
||||||
|
"type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
var processMetadataBytesTestCases = []struct {
|
||||||
|
caseId int
|
||||||
|
request H
|
||||||
|
existing H
|
||||||
|
want H
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
101,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
102,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
103,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "request",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
104,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "request",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
105,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "existing",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "existing",
|
||||||
|
},
|
||||||
|
H{},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
107,
|
||||||
|
H{
|
||||||
|
"User-Agent": "firefox",
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging": "A=B&a=b&type=request",
|
||||||
|
headers.AmzUserMetaDirective: DirectiveReplace,
|
||||||
|
headers.AmzObjectTaggingDirective: DirectiveReplace,
|
||||||
|
},
|
||||||
|
H{},
|
||||||
|
H{
|
||||||
|
"X-Amz-Meta-My-Meta": "request",
|
||||||
|
"X-Amz-Tagging-A": "B",
|
||||||
|
"X-Amz-Tagging-a": "b",
|
||||||
|
"X-Amz-Tagging-type": "request",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessMetadata(t *testing.T) {
|
||||||
|
for _, tc := range processMetadataTestCases {
|
||||||
|
reqHeader := transferHToHeader(tc.request)
|
||||||
|
existing := transferHToHeader(tc.existing)
|
||||||
|
replaceMeta, replaceTagging := replaceDirective(reqHeader)
|
||||||
|
|
||||||
|
err := processMetadata(reqHeader, existing, replaceMeta, replaceTagging, func(_ string, _ string) (tags map[string]string, err error) {
|
||||||
|
return tc.getTags, nil
|
||||||
|
}, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := transferHeaderToH(reqHeader)
|
||||||
|
fmtTagging(result, tc.want)
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(result, tc.want) {
|
||||||
|
t.Error(fmt.Errorf("\n### CaseID: %d ###"+
|
||||||
|
"\nRequest:%v"+
|
||||||
|
"\nExisting:%v"+
|
||||||
|
"\nGetTags:%v"+
|
||||||
|
"\nWant:%v"+
|
||||||
|
"\nActual:%v",
|
||||||
|
tc.caseId, tc.request, tc.existing, tc.getTags, tc.want, result))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessMetadataBytes(t *testing.T) {
|
||||||
|
for _, tc := range processMetadataBytesTestCases {
|
||||||
|
reqHeader := transferHToHeader(tc.request)
|
||||||
|
existing := transferHToBytesArr(tc.existing)
|
||||||
|
replaceMeta, replaceTagging := replaceDirective(reqHeader)
|
||||||
|
extends := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging)
|
||||||
|
|
||||||
|
result := transferBytesArrToH(extends)
|
||||||
|
fmtTagging(result, tc.want)
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(result, tc.want) {
|
||||||
|
t.Error(fmt.Errorf("\n### CaseID: %d ###"+
|
||||||
|
"\nRequest:%v"+
|
||||||
|
"\nExisting:%v"+
|
||||||
|
"\nWant:%v"+
|
||||||
|
"\nActual:%v",
|
||||||
|
tc.caseId, tc.request, tc.existing, tc.want, result))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fmtTagging(maps ...map[string]string) {
|
||||||
|
for _, m := range maps {
|
||||||
|
if tagging := m[headers.AmzObjectTagging]; len(tagging) > 0 {
|
||||||
|
split := strings.Split(tagging, "&")
|
||||||
|
sort.Strings(split)
|
||||||
|
m[headers.AmzObjectTagging] = strings.Join(split, "&")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func transferHToHeader(data map[string]string) http.Header {
|
||||||
|
header := http.Header{}
|
||||||
|
for k, v := range data {
|
||||||
|
header.Add(k, v)
|
||||||
|
}
|
||||||
|
return header
|
||||||
|
}
|
||||||
|
|
||||||
|
func transferHToBytesArr(data map[string]string) map[string][]byte {
|
||||||
|
m := make(map[string][]byte, len(data))
|
||||||
|
for k, v := range data {
|
||||||
|
m[k] = []byte(v)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func transferBytesArrToH(data map[string][]byte) H {
|
||||||
|
m := make(map[string]string, len(data))
|
||||||
|
for k, v := range data {
|
||||||
|
m[k] = string(v)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func transferHeaderToH(data map[string][]string) H {
|
||||||
|
m := make(map[string]string, len(data))
|
||||||
|
for k, v := range data {
|
||||||
|
m[k] = v[len(v)-1]
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
Loading…
Reference in a new issue