gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #25 from chrislusf/master

Browse source 
sync
This commit is contained in:
hilimd 2020-10-12 11:01:13 +08:00 committed by GitHub
parent b9a446839a d5fcb0f474
commit 2ff727a32d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 98 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -60,6 +60,7 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
github.com/seaweedfs/goexif v1.0.2
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect

2
go.sum
View file

@ -452,6 +452,8 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=

2
other/java/client/pom.xml
View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.4.8</version>
<version>1.4.9</version>
<parent>
<groupId>org.sonatype.oss</groupId>

2
other/java/client/pom.xml.deploy
View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.4.8</version>
<version>1.4.9</version>
<parent>
<groupId>org.sonatype.oss</groupId>

2
other/java/client/pom_debug.xml
View file

@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>1.4.8</version>
<version>1.4.9</version>
<parent>
<groupId>org.sonatype.oss</groupId>

12
other/java/client/src/main/java/seaweedfs/client/FilerClient.java
View file

@ -1,5 +1,6 @@
package seaweedfs.client;
import com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -272,15 +273,20 @@ public class FilerClient {
public boolean createEntry(String parent, FilerProto.Entry entry) {
try {
filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
FilerProto.CreateEntryResponse createEntryResponse =
filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
.setDirectory(parent)
.setEntry(entry)
.build());
if (Strings.isNullOrEmpty(createEntryResponse.getError())) {
return true;
}
LOG.warn("createEntry {}/{} error: {}", parent, entry.getName(), createEntryResponse.getError());
return false;
} catch (Exception e) {
LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
return false;
}
return true;
}
public boolean updateEntry(String parent, FilerProto.Entry entry) {
@ -290,7 +296,7 @@ public class FilerClient {
.setEntry(entry)
.build());
} catch (Exception e) {
LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
LOG.warn("updateEntry {}/{}: {}", parent, entry.getName(), e);
return false;
}
return true;

2
other/java/hdfs2/dependency-reduced-pom.xml
View file

@ -301,7 +301,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>1.4.8</seaweedfs.client.version>
<seaweedfs.client.version>1.4.9</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>
</project>

2
other/java/hdfs2/pom.xml
View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>1.4.8</seaweedfs.client.version>
<seaweedfs.client.version>1.4.9</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>

2
other/java/hdfs3/dependency-reduced-pom.xml
View file

@ -309,7 +309,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>1.4.8</seaweedfs.client.version>
<seaweedfs.client.version>1.4.9</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>
</project>

2
other/java/hdfs3/pom.xml
View file

@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>1.4.8</seaweedfs.client.version>
<seaweedfs.client.version>1.4.9</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>

14
weed/command/mount.go
View file

@ -2,6 +2,7 @@ package command
import (
"os"
"time"
)
type MountOptions struct {
@ -25,9 +26,10 @@ type MountOptions struct {
}
var (
mountOptions MountOptions
mountCpuProfile *string
mountMemProfile *string
mountOptions MountOptions
mountCpuProfile *string
mountMemProfile *string
mountReadRetryTime *time.Duration
)
func init() {
@ -46,11 +48,13 @@ func init() {
mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access the file system")
mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
}
var cmdMount = &Command{

5
weed/command/mount_std.go
View file

@ -5,6 +5,7 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filesys/meta_cache"
"os"
"os/user"
@ -29,6 +30,10 @@ import (
func runMount(cmd *Command, args []string) bool {
grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
if *mountReadRetryTime < time.Second {
*mountReadRetryTime = time.Second
}
filer.ReadWaitTime = *mountReadRetryTime
umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64)
if umaskErr != nil {

2
weed/filer/filechunk_manifest.go
View file

@ -98,7 +98,7 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
var err error
var buffer bytes.Buffer
for waitTime := time.Second; waitTime < 10*time.Second; waitTime += waitTime / 2 {
for waitTime := time.Second; waitTime < ReadWaitTime; waitTime += waitTime / 2 {
for _, urlString := range urlStrings {
err = util.ReadUrlAsStream(urlString, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
buffer.Write(data)

18
weed/filer/reader_at.go
View file

@ -11,6 +11,11 @@ import (
"io"
"math/rand"
"sync"
"time"
)
var (
ReadWaitTime = 6 * time.Second
)
type ChunkReadAt struct {
@ -37,7 +42,8 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType {
vid := VolumeId(fileId)
locations, found := vidCache[vid]
if !found {
waitTime := time.Second
for !found && waitTime < ReadWaitTime {
// println("looking up volume", vid)
err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
@ -56,6 +62,16 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType {
return nil
})
if err == nil {
break
}
glog.V(1).Infof("wait for volume %s", vid)
time.Sleep(waitTime)
waitTime += waitTime / 2
}
if err != nil {
return nil, err
}
for _, loc := range locations.Locations {

2
weed/filer/stream.go
View file

@ -34,7 +34,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
urlStrings := fileId2Url[chunkView.FileId]
data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
if err == nil {
if err != nil {
return err
}
w.Write(data)

2
weed/filesys/filehandle.go
View file

@ -79,7 +79,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
if err != nil {
glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err)
return nil
return fuse.EIO
}
if totalRead > int64(len(buff)) {

44
weed/server/filer_server_handlers_read_dir.go
View file

@ -2,6 +2,9 @@ package weed_server
import (
"context"
"encoding/base64"
"fmt"
"github.com/skip2/go-qrcode"
"net/http"
"strconv"
"strings"
@ -65,21 +68,30 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
lastFileName,
shouldDisplayLoadMore,
})
} else {
ui.StatusTpl.Execute(w, struct {
Path string
Breadcrumbs []ui.Breadcrumb
Entries interface{}
Limit int
LastFileName string
ShouldDisplayLoadMore bool
}{
path,
ui.ToBreadcrumb(path),
entries,
limit,
lastFileName,
shouldDisplayLoadMore,
})
return
}
var qrImageString string
img, err := qrcode.Encode(fmt.Sprintf("http://%s:%d%s", fs.option.Host, fs.option.Port, r.URL.Path), qrcode.Medium, 128)
if err == nil {
qrImageString = base64.StdEncoding.EncodeToString(img)
}
ui.StatusTpl.Execute(w, struct {
Path string
Breadcrumbs []ui.Breadcrumb
Entries interface{}
Limit int
LastFileName string
ShouldDisplayLoadMore bool
QrImage string
}{
path,
ui.ToBreadcrumb(path),
entries,
limit,
lastFileName,
shouldDisplayLoadMore,
qrImageString,
})
}

19
weed/server/filer_ui/templates.go
View file

@ -21,9 +21,11 @@ var funcMap = template.FuncMap{
var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOCTYPE html>
<html>
<head>
<title>SeaweedFS Filer</title>
<link rel="stylesheet" href="/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css">
<title>SeaweedFS Filer</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css">
<style>
body { padding-bottom: 70px; }
#drop-area {
border: 1px transparent;
}
@ -46,6 +48,11 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
#fileElem {
display: none;
}
.qrImage {
display: block;
margin-left: auto;
margin-right: auto;
}
</style>
</head>
<body>
@ -116,6 +123,14 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
</a>
</div>
{{end}}
<br/>
<br/>
<div class="navbar navbar-fixed-bottom">
<img src="data:image/png;base64,{{.QrImage}}" class="qrImage" />
</div>
</div>
</body>
<script type="text/javascript">