diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 000000000..47a677e6d --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,22 @@ +name: Cleanup + +on: + push: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + + steps: + + - name: Delete old release assets + uses: mknejp/delete-release-assets@v1 + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7439fb604..0681f3dac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: matrix: goos: [linux, windows, darwin, freebsd ] goarch: [amd64, arm] - exclude: + exclude: - goarch: arm goos: darwin - goarch: arm @@ -21,46 +21,42 @@ jobs: steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code into the Go module directory + uses: actions/checkout@v2 - - name: Delete old release assets - uses: mknejp/delete-release-assets@v1 - with: - token: ${{ github.token }} - tag: dev - fail-if-no-assets: false - assets: | - weed-* + - name: Wait for the deletion + uses: jakejarvis/wait-action@master + with: + time: '30s' - - name: Set BUILD_TIME env - run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.14 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - build_flags: -tags 5BytesOffset # optional, default is - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed-large-disk - asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.14 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.14 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed- - asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.14 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed- + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" diff --git a/docker/Dockerfile b/docker/Dockerfile index 528a52609..6309e7e2e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,7 @@ FROM alpine +ARG RELEASE=latest # 'latest' or 'dev' + RUN \ ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \ elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \ @@ -13,7 +15,7 @@ RUN \ # Install SeaweedFS and Supercronic ( for cron job mode ) apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \ apk add fuse && \ - wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \ + wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \ tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ diff --git a/docker/compose/local-mount-compose.yml b/docker/compose/local-mount-compose.yml index 4361d6d99..b1c579cdf 100644 --- a/docker/compose/local-mount-compose.yml +++ b/docker/compose/local-mount-compose.yml @@ -30,7 +30,7 @@ services: mount_1: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1" -volumeServerAccess=filerProxy' + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"' depends_on: - master - volume @@ -38,7 +38,7 @@ services: mount_2: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1" -volumeServerAcess=publicUrl' + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1 -volumeServerAcess=publicUrl"' depends_on: - master - volume diff --git a/weed/command/filer.go b/weed/command/filer.go index 7f665cec8..6660bd694 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -52,7 +52,7 @@ type FilerOptions struct { func init() { cmdFiler.Run = runFiler // break init cycle f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") - f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") + f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection") f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address") f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") @@ -83,6 +83,8 @@ func init() { filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port") filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files") + filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files") + filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd] hard drive or solid state drive") filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/server.go b/weed/command/server.go index 61cac2cc7..d7c41b014 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -121,6 +121,8 @@ func init() { webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") + webdavOptions.replication = cmdServer.Flag.String("webdav.replication", "", "replication to create the files") + webdavOptions.disk = cmdServer.Flag.String("webdav.disk", "", "[hdd|ssd] hard drive or solid state drive") webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index dc84b1fd0..3e4532d6e 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -25,6 +25,8 @@ type WebDavOption struct { filer *string port *int collection *string + replication *string + disk *string tlsPrivateKey *string tlsCertificate *string cacheDir *string @@ -36,6 +38,8 @@ func init() { webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address") webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port") webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files") + webDavStandaloneOptions.replication = cmdWebDav.Flag.String("replication", "", "replication to create the files") + webDavStandaloneOptions.disk = cmdWebDav.Flag.String("disk", "", "[hdd|ssd] hard drive or solid state drive") webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file") webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") @@ -107,6 +111,8 @@ func (wo *WebDavOption) startWebDav() bool { FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, Collection: *wo.collection, + Replication: *wo.replication, + DiskType: *wo.disk, Uid: uid, Gid: gid, Cipher: cipher, diff --git a/weed/filer.toml b/weed/filer.toml deleted file mode 100644 index c0530b3f1..000000000 --- a/weed/filer.toml +++ /dev/null @@ -1,11 +0,0 @@ -[elastic7] -enabled = true -servers = [ - "http://localhost:9200", -] -username = "" -password = "" -sniff_enabled = false -healthcheck_enabled = false -# increase the value is recommend, be sure the value in Elastic is greater or equal here -index.max_result_window = 10000 diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index b4f0e5890..8e549f5ad 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -46,17 +46,19 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) { return fc.LoadFromBytes(entry.Content) } - return fc.loadFromChunks(filer, entry.Chunks) + return fc.loadFromChunks(filer, entry.Content, entry.Chunks) } -func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) (err error) { - data, err := filer.readEntry(chunks) - if err != nil { - glog.Errorf("read filer conf content: %v", err) - return +func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) { + if len(content) == 0 { + content, err = filer.readEntry(chunks) + if err != nil { + glog.Errorf("read filer conf content: %v", err) + return + } } - return fc.LoadFromBytes(data) + return fc.LoadFromBytes(content) } func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index d93a7e9e3..295a5039e 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -40,7 +40,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { fc := NewFilerConf() - err := fc.loadFromChunks(f, entry.Chunks) + err := fc.loadFromChunks(f, entry.Content, entry.Chunks) if err != nil { glog.Errorf("read filer conf chunks: %v", err) return diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index ec49d177b..910114313 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -64,7 +64,6 @@ func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*gr grpc.WithDefaultCallOptions( grpc.MaxCallSendMsgSize(Max_Message_Size), grpc.MaxCallRecvMsgSize(Max_Message_Size), - grpc.WaitForReady(true), ), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, // client ping server if no activity for this long diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 1699eaa36..29aff5c0b 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -77,7 +77,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest if !ms.Topo.HasWritableVolume(option) { if ms.Topo.AvailableSpaceFor(option) <= 0 { - return nil, fmt.Errorf("No free volumes left!") + return nil, fmt.Errorf("no free volumes left for "+option.String()) } ms.vgLock.Lock() if !ms.Topo.HasWritableVolume(option) { diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 117423ec6..a9fecc5bd 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -113,7 +113,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) if !ms.Topo.HasWritableVolume(option) { if ms.Topo.AvailableSpaceFor(option) <= 0 { - writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left!"}) + writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left for " + option.String()}) return } ms.vgLock.Lock() diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 4b57c7afe..c3f68fdee 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -33,6 +33,7 @@ type WebDavOption struct { BucketsPath string GrpcDialOption grpc.DialOption Collection string + Replication string DiskType string Uid uint32 Gid uint32 @@ -225,7 +226,7 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f Uid: fs.option.Uid, Gid: fs.option.Gid, Collection: fs.option.Collection, - Replication: "000", + Replication: fs.option.Replication, TtlSec: 0, }, }, @@ -381,7 +382,7 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64 request := &filer_pb.AssignVolumeRequest{ Count: 1, - Replication: "", + Replication: f.fs.option.Replication, Collection: f.fs.option.Collection, DiskType: f.fs.option.DiskType, Path: name, diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index 3c6c31eac..27c6815d4 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -20,11 +20,11 @@ type commandVolumeTierMove struct { } func (c *commandVolumeTierMove) Name() string { - return "volume.tier.upload" + return "volume.tier.move" } func (c *commandVolumeTierMove) Help() string { - return `change a volume from one disk type to another + return ` change a volume from one disk type to another volume.tier.move -source=hdd -target=ssd [-collection=""] [-fullPercent=95] [-quietFor=1h] volume.tier.move -target=hdd [-collection=""] -volumeId= diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 2d5166acf..00adb1abc 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -92,7 +92,7 @@ func processEachCmd(reg *regexp.Regexp, cmd string, commandEnv *CommandEnv) bool func printGenericHelp() { msg := - `Type: "help " for help on + `Type: "help " for help on . Most commands support " -h" also for options. ` fmt.Print(msg) diff --git a/weed/storage/super_block/replica_placement.go b/weed/storage/super_block/replica_placement.go index fcccbba7d..65ec53819 100644 --- a/weed/storage/super_block/replica_placement.go +++ b/weed/storage/super_block/replica_placement.go @@ -6,9 +6,9 @@ import ( ) type ReplicaPlacement struct { - SameRackCount int - DiffRackCount int - DiffDataCenterCount int + SameRackCount int `json:"node,omitempty"` + DiffRackCount int `json:"rack,omitempty"` + DiffDataCenterCount int `json:"dc,omitempty"` } func NewReplicaPlacementFromString(t string) (*ReplicaPlacement, error) { diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 1a0ebf761..69f739dd5 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -207,7 +207,26 @@ func (dn *DataNode) ToMap() interface{} { ret := make(map[string]interface{}) ret["Url"] = dn.Url() ret["PublicUrl"] = dn.PublicUrl - ret["Disks"] = dn.diskUsages.ToMap() + + // aggregated volume info + var volumeCount, ecShardCount, maxVolumeCount int64 + var volumeIds string + for _, diskUsage := range dn.diskUsages.usages { + volumeCount += diskUsage.volumeCount + ecShardCount += diskUsage.ecShardCount + maxVolumeCount += diskUsage.maxVolumeCount + } + + for _, disk := range dn.Children() { + d := disk.(*Disk) + volumeIds += " " + d.GetVolumeIds() + } + + ret["Volumes"] = volumeCount + ret["EcShards"] = ecShardCount + ret["Max"] = maxVolumeCount + ret["VolumeIds"] = volumeIds + return ret } diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 37d5e1272..a085f8dff 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -58,16 +58,6 @@ func (d *DiskUsages) negative() *DiskUsages { return t } -func (d *DiskUsages) ToMap() interface{} { - d.RLock() - defer d.RUnlock() - ret := make(map[string]interface{}) - for diskType, diskUsage := range d.usages { - ret[diskType.String()] = diskUsage.ToMap() - } - return ret -} - func (d *DiskUsages) ToDiskInfo() map[string]*master_pb.DiskInfo { ret := make(map[string]*master_pb.DiskInfo) for diskType, diskUsageCounts := range d.usages { @@ -135,15 +125,6 @@ func (a *DiskUsageCounts) minus(b *DiskUsageCounts) *DiskUsageCounts { } } -func (diskUsage *DiskUsageCounts) ToMap() interface{} { - ret := make(map[string]interface{}) - ret["Volumes"] = diskUsage.volumeCount - ret["EcShards"] = diskUsage.ecShardCount - ret["Max"] = diskUsage.maxVolumeCount - ret["Free"] = diskUsage.FreeSpace() - return ret -} - func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { du.Lock() defer du.Unlock() diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index 4b0a4837e..8941a049b 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -1,6 +1,7 @@ package topology import ( + "encoding/json" "fmt" "github.com/chrislusf/seaweedfs/weed/storage/types" "math/rand" @@ -25,15 +26,15 @@ This package is created to resolve these replica placement issues: */ type VolumeGrowOption struct { - Collection string - ReplicaPlacement *super_block.ReplicaPlacement - Ttl *needle.TTL - DiskType types.DiskType - Prealloacte int64 - DataCenter string - Rack string - DataNode string - MemoryMapMaxSizeMb uint32 + Collection string `json:"collection,omitempty"` + ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"` + Ttl *needle.TTL `json:"ttl,omitempty"` + DiskType types.DiskType `json:"disk,omitempty"` + Prealloacte int64 `json:"prealloacte,omitempty"` + DataCenter string `json:"dataCenter,omitempty"` + Rack string `json:"rack,omitempty"` + DataNode string `json:"dataNode,omitempty"` + MemoryMapMaxSizeMb uint32 `json:"memoryMapMaxSizeMb,omitempty"` } type VolumeGrowth struct { @@ -41,7 +42,8 @@ type VolumeGrowth struct { } func (o *VolumeGrowOption) String() string { - return fmt.Sprintf("Collection:%s, ReplicaPlacement:%v, Ttl:%v, DataCenter:%s, Rack:%s, DataNode:%s", o.Collection, o.ReplicaPlacement, o.Ttl, o.DataCenter, o.Rack, o.DataNode) + blob, _ := json.Marshal(o) + return string(blob) } func NewDefaultVolumeGrowth() *VolumeGrowth {