Merge pull request #35 from chrislusf/master

sync
This commit is contained in:
hilimd 2020-11-11 10:16:39 +08:00 committed by GitHub
commit a38efe2e7b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 2028 additions and 1390 deletions

View file

@ -144,9 +144,9 @@ On top of the object store, optional [Filer] can support directories and POSIX a
## Quick Start ## ## Quick Start ##
* Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe` * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
* Run `weed server -dir=. -s3` to start one master, one volume server, one filer, and one S3 gateway, with data stored in current directory. * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
You can also start each module individually. See `weed -h`, or `weed <command> -h` for help. Also, to increase capacity, just add more volume servers by `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally or a different machine. That is it!
## Example: Using Seaweed Object Store ## ## Example: Using Seaweed Object Store ##

View file

@ -1,4 +1,4 @@
apiVersion: v1 apiVersion: v1
description: SeaweedFS description: SeaweedFS
name: seaweedfs name: seaweedfs
version: 2.08 version: 2.09

File diff suppressed because it is too large Load diff

View file

@ -101,7 +101,7 @@ spec:
-port={{ .Values.filer.port }} \ -port={{ .Values.filer.port }} \
{{- if .Values.filer.metricsPort }} {{- if .Values.filer.metricsPort }}
-metricsPort {{ .Values.filer.metricsPort }} \ -metricsPort {{ .Values.filer.metricsPort }} \
{{- end }}} {{- end }}
{{- if .Values.filer.redirectOnRead }} {{- if .Values.filer.redirectOnRead }}
-redirectOnRead \ -redirectOnRead \
{{- end }} {{- end }}

View file

@ -73,7 +73,7 @@ spec:
-port={{ .Values.s3.port }} \ -port={{ .Values.s3.port }} \
{{- if .Values.s3.metricsPort }} {{- if .Values.s3.metricsPort }}
-metricsPort {{ .Values.s3.metricsPort }} \ -metricsPort {{ .Values.s3.metricsPort }} \
{{- end }}} {{- end }}
{{- if .Values.global.enableSecurity }} {{- if .Values.global.enableSecurity }}
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \ -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-key.file=/usr/local/share/ca-certificates/client/tls.key \ -key.file=/usr/local/share/ca-certificates/client/tls.key \

View file

@ -14,10 +14,10 @@ spec:
protocol: TCP protocol: TCP
{{- if .Values.s3.metricsPort }} {{- if .Values.s3.metricsPort }}
- name: "swfs-s3-metrics" - name: "swfs-s3-metrics"
port: {{ .Values.filer.s3 }} port: {{ .Values.s3.metricsPort }}
targetPort: {{ .Values.s3.metricsPort }} targetPort: {{ .Values.s3.metricsPort }}
protocol: TCP protocol: TCP
{{- end }}} {{- end }}
selector: selector:
app: {{ template "seaweedfs.name" . }} app: {{ template "seaweedfs.name" . }}
component: s3 component: s3

View file

@ -15,4 +15,4 @@ spec:
selector: selector:
app: {{ template "seaweedfs.name" . }} app: {{ template "seaweedfs.name" . }}
component: s3 component: s3
{{- end }}} {{- end }}

View file

@ -0,0 +1,20 @@
{{- if .Values.global.monitoring.enabled }}
{{- $files := .Files.Glob "dashboards/*.json" }}
{{- if $files }}
apiVersion: v1
kind: ConfigMapList
items:
{{- range $path, $fileContents := $files }}
{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }}
- apiVersion: v1
kind: ConfigMap
metadata:
name: {{ printf "%s" $dashboardName | lower | replace "_" "-" }}
namespace: {{ $.Release.Namespace }}
labels:
grafana_dashboard: "1"
data:
{{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }}
{{- end }}
{{- end }}
{{- end }}

File diff suppressed because it is too large Load diff

View file

@ -22,7 +22,7 @@ spec:
port: {{ .Values.volume.metricsPort }} port: {{ .Values.volume.metricsPort }}
targetPort: {{ .Values.volume.metricsPort }} targetPort: {{ .Values.volume.metricsPort }}
protocol: TCP protocol: TCP
{{- end }}} {{- end }}
selector: selector:
app: {{ template "seaweedfs.name" . }} app: {{ template "seaweedfs.name" . }}
component: volume component: volume

View file

@ -15,4 +15,4 @@ spec:
selector: selector:
app: {{ template "seaweedfs.name" . }} app: {{ template "seaweedfs.name" . }}
component: volume component: volume
{{- end }}} {{- end }}

View file

@ -78,7 +78,7 @@ spec:
-port={{ .Values.volume.port }} \ -port={{ .Values.volume.port }} \
{{- if .Values.volume.metricsPort }} {{- if .Values.volume.metricsPort }}
-metricsPort {{ .Values.volume.metricsPort }} \ -metricsPort {{ .Values.volume.metricsPort }} \
{{- end }}} {{- end }}
-dir={{ .Values.volume.dir }} \ -dir={{ .Values.volume.dir }} \
-max={{ .Values.volume.maxVolumes }} \ -max={{ .Values.volume.maxVolumes }} \
{{- if .Values.volume.rack }} {{- if .Values.volume.rack }}

View file

@ -4,7 +4,7 @@ global:
registry: "" registry: ""
repository: "" repository: ""
imageName: chrislusf/seaweedfs imageName: chrislusf/seaweedfs
imageTag: "2.08" imageTag: "2.09"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret imagePullSecrets: imagepullsecret
restartPolicy: Always restartPolicy: Always

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version> <version>1.5.3</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version> <version>1.5.3</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId> <groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId> <artifactId>seaweedfs-client</artifactId>
<version>1.5.2</version> <version>1.5.3</version>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>

View file

@ -301,7 +301,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version> <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version> <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version> <hadoop.version>2.9.2</hadoop.version>
</properties> </properties>

View file

@ -309,7 +309,7 @@
</snapshotRepository> </snapshotRepository>
</distributionManagement> </distributionManagement>
<properties> <properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version> <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>
</project> </project>

View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<seaweedfs.client.version>1.5.2</seaweedfs.client.version> <seaweedfs.client.version>1.5.3</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version> <hadoop.version>3.1.1</hadoop.version>
</properties> </properties>

View file

@ -38,10 +38,12 @@ func init() {
func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) { func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
var bytes []byte var bytes []byte
if r.FormValue("pretty") != "" { if obj != nil {
bytes, err = json.MarshalIndent(obj, "", " ") if r.FormValue("pretty") != "" {
} else { bytes, err = json.MarshalIndent(obj, "", " ")
bytes, err = json.Marshal(obj) } else {
bytes, err = json.Marshal(obj)
}
} }
if err != nil { if err != nil {
return return

View file

@ -26,11 +26,19 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("head").Observe(time.Since(start).Seconds())
case "DELETE": case "DELETE":
stats.FilerRequestCounter.WithLabelValues("delete").Inc() stats.FilerRequestCounter.WithLabelValues("delete").Inc()
fs.DeleteHandler(w, r) if _, ok := r.URL.Query()["tagging"]; ok {
fs.DeleteTaggingHandler(w,r)
} else {
fs.DeleteHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
case "PUT": case "PUT":
stats.FilerRequestCounter.WithLabelValues("put").Inc() stats.FilerRequestCounter.WithLabelValues("put").Inc()
fs.PostHandler(w, r) if _, ok := r.URL.Query()["tagging"]; ok {
fs.PutTaggingHandler(w,r)
} else {
fs.PostHandler(w, r)
}
stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("put").Observe(time.Since(start).Seconds())
case "POST": case "POST":
stats.FilerRequestCounter.WithLabelValues("post").Inc() stats.FilerRequestCounter.WithLabelValues("post").Inc()

View file

@ -0,0 +1,102 @@
package weed_server
import (
"context"
"net/http"
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
)
// add or replace one file Seaweed- prefixed attributes
// curl -X PUT -H "Seaweed-Name1: value1" http://localhost:8888/path/to/a/file?tagging
func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
path := r.URL.Path
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
}
existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
if err != nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry == nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry.Extended == nil {
existingEntry.Extended = make(map[string][]byte)
}
for header, values := range r.Header {
if strings.HasPrefix(header, needle.PairNamePrefix) {
for _, value := range values {
existingEntry.Extended[header] = []byte(value)
}
}
}
if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
glog.V(0).Infof("failing to update %s tagging : %v", path, dbErr)
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
writeJsonQuiet(w, r, http.StatusAccepted, nil)
return
}
// remove all Seaweed- prefixed attributes
// curl -X DELETE http://localhost:8888/path/to/a/file?tagging
func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
path := r.URL.Path
if strings.HasSuffix(path, "/") {
path = path[:len(path)-1]
}
existingEntry, err := fs.filer.FindEntry(ctx, util.FullPath(path))
if err != nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry == nil {
writeJsonError(w, r, http.StatusNotFound, err)
return
}
if existingEntry.Extended == nil {
existingEntry.Extended = make(map[string][]byte)
}
hasDeletion := false
for header, _ := range existingEntry.Extended {
if strings.HasPrefix(header, needle.PairNamePrefix) {
delete(existingEntry.Extended, header)
hasDeletion = true
}
}
if !hasDeletion {
writeJsonQuiet(w, r, http.StatusNotModified, nil)
return
}
if dbErr := fs.filer.CreateEntry(ctx, existingEntry, false, false, nil); dbErr != nil {
glog.V(0).Infof("failing to delete %s tagging : %v", path, dbErr)
writeJsonError(w, r, http.StatusInternalServerError, err)
return
}
writeJsonQuiet(w, r, http.StatusAccepted, nil)
return
}

View file

@ -369,18 +369,20 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st
func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica { func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
allSame := true sort.Slice(replicas, func(i, j int) bool {
oldest := replicas[0] a, b := replicas[i], replicas[j]
for _, replica := range replicas { if a.info.CompactRevision != b.info.CompactRevision {
if replica.info.ModifiedAtSecond < oldest.info.ModifiedAtSecond { return a.info.CompactRevision < b.info.CompactRevision
oldest = replica
allSame = false
} }
} if a.info.ModifiedAtSecond != b.info.ModifiedAtSecond {
if !allSame { return a.info.ModifiedAtSecond < b.info.ModifiedAtSecond
return oldest }
} if a.info.Size != b.info.Size {
return a.info.Size < b.info.Size
}
return false
})
return replicas[0]
// TODO what if all the replicas have the same timestamp?
return oldest
} }

View file

@ -472,7 +472,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
maxVolumeCount += int(uint64(unclaimedSpaces)/volumeSizeLimit) - 1 maxVolumeCount += int(uint64(unclaimedSpaces)/volumeSizeLimit) - 1
} }
diskLocation.MaxVolumeCount = maxVolumeCount diskLocation.MaxVolumeCount = maxVolumeCount
glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB", glog.V(2).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024) diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
hasChanges = hasChanges || currentMaxVolumeCount != diskLocation.MaxVolumeCount hasChanges = hasChanges || currentMaxVolumeCount != diskLocation.MaxVolumeCount
} }