gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into add_s3

Browse source
This commit is contained in:
Chris Lu 2018-07-21 15:59:18 -07:00
parent 8480008a9a c98df05ed0
commit dc8dd8b486
5 changed files with 200 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
weed/filesys/dir.go
View file

@ -4,17 +4,20 @@ import (
"context"
"os"
"path"
"path/filepath"
"time"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"path/filepath"
"time"
"github.com/chrislusf/seaweedfs/weed/filer2"
)
type Dir struct {
Path string
wfs *WFS
Path string
wfs *WFS
attributes *filer_pb.FuseAttributes
}
var _ = fs.Node(&Dir{})
@ -24,6 +27,7 @@ var _ = fs.NodeRequestLookuper(&Dir{})
var _ = fs.HandleReadDirAller(&Dir{})
var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})
var _ = fs.NodeSetattrer(&Dir{})
func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
@ -48,8 +52,6 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
parent, name := filepath.Split(dir.Path)
var attributes *filer_pb.FuseAttributes
err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.GetEntryAttributesRequest{
@ -64,7 +66,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return err
}
attributes = resp.Attributes
dir.attributes = resp.Attributes
return nil
})
@ -76,26 +78,26 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
attr.Mode = os.FileMode(attributes.FileMode) | os.ModeDir
if dir.Path == "/" && attributes.FileMode == 0 {
attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
if dir.Path == "/" && dir.attributes.FileMode == 0 {
attr.Valid = time.Second
}
attr.Mtime = time.Unix(attributes.Mtime, 0)
attr.Ctime = time.Unix(attributes.Crtime, 0)
attr.Gid = attributes.Gid
attr.Uid = attributes.Uid
attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
attr.Gid = dir.attributes.Gid
attr.Uid = dir.attributes.Uid
return nil
}
func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk, attr *filer_pb.FuseAttributes) *File {
return &File{
Name: name,
dir: dir,
wfs: dir.wfs,
// attributes: &filer_pb.FuseAttributes{},
Chunks: chunks,
Name: name,
dir: dir,
wfs: dir.wfs,
attributes: attr,
Chunks: chunks,
}
}
@ -132,7 +134,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
})
if err == nil {
file := dir.newFile(req.Name, nil)
file := dir.newFile(req.Name, nil, &filer_pb.FuseAttributes{})
file.isOpen = true
return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
}
@ -200,9 +202,9 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
if entry != nil {
if entry.IsDirectory {
node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
} else {
node = dir.newFile(req.Name, entry.Chunks)
node = dir.newFile(req.Name, entry.Chunks, entry.Attributes)
}
resp.EntryValid = time.Duration(0)
@ -272,3 +274,45 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
})
}
func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
if req.Valid.Mode() {
dir.attributes.FileMode = uint32(req.Mode)
}
if req.Valid.Uid() {
dir.attributes.Uid = req.Uid
}
if req.Valid.Gid() {
dir.attributes.Gid = req.Gid
}
if req.Valid.Mtime() {
dir.attributes.Mtime = req.Mtime.Unix()
}
parentDir, name := filer2.FullPath(dir.Path).DirAndName()
return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: parentDir,
Entry: &filer_pb.Entry{
Name: name,
Attributes: dir.attributes,
},
}
glog.V(1).Infof("set attr directory entry: %v", request)
_, err := client.UpdateEntry(ctx, request)
if err != nil {
glog.V(0).Infof("UpdateEntry %s: %v", dir.Path, err)
return fuse.EIO
}
return nil
})
}

23
weed/filesys/file.go
View file

@ -34,7 +34,7 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
if file.attributes == nil || !file.isOpen {
item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
if item != nil && !item.Expired(){
if item != nil && !item.Expired() {
entry := item.Value().(*filer_pb.Entry)
file.Chunks = entry.Chunks
file.attributes = entry.Attributes
@ -121,7 +121,26 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
file.attributes.Mtime = req.Mtime.Unix()
}
return nil
return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: file.dir.Path,
Entry: &filer_pb.Entry{
Name: file.Name,
Attributes: file.attributes,
Chunks: file.Chunks,
},
}
glog.V(1).Infof("set attr file entry: %v", request)
_, err := client.UpdateEntry(ctx, request)
if err != nil {
glog.V(0).Infof("UpdateEntry file %s/%s: %v", file.dir.Path, file.Name, err)
return fuse.EIO
}
return nil
})
}

2
weed/server/master_ui/templates.go
View file

@ -39,7 +39,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
<td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
</tr>
<tr>
<td class="col-sm-2 field-label"><label>Peers:</label></td>
<td class="col-sm-2 field-label"><label>Other Masters:</label></td>
<td class="col-sm-10"><ul class="list-unstyled">
{{ range $k, $p := .Peers }}
<li><a href="{{ $p.ConnectionString }}">{{ $p.Name }}</a></li>

99
weed/storage/needle.go
View file

@ -3,18 +3,14 @@ package storage
import (
"encoding/json"
"fmt"
"io/ioutil"
"mime"
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/images"
"github.com/chrislusf/seaweedfs/weed/operation"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"io/ioutil"
)
const (
@ -62,91 +58,20 @@ func ParseUpload(r *http.Request) (
}
}
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
return
}
//first multi-part item
part, fe := form.NextPart()
if fe != nil {
glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
e = fe
return
}
fileName = part.FileName()
if fileName != "" {
fileName = path.Base(fileName)
}
data, e = ioutil.ReadAll(part)
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
return
}
//if the filename is empty string, do a search on the other multi-part items
for fileName == "" {
part2, fe := form.NextPart()
if fe != nil {
break // no more or on error, just safely break
}
fName := part2.FileName()
//found the first <file type> multi-part has filename
if fName != "" {
data2, fe2 := ioutil.ReadAll(part2)
if fe2 != nil {
glog.V(0).Infoln("Reading Content [ERROR]", fe2)
e = fe2
return
}
//update
data = data2
fileName = path.Base(fName)
break
}
}
isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
if !isChunkedFile {
dotIndex := strings.LastIndex(fileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(fileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && mtype != contentType {
mimeType = contentType //only return mime type if not deductable
mtype = contentType
}
if part.Header.Get("Content-Encoding") == "gzip" {
isGzipped = true
} else if operation.IsGzippable(ext, mtype) {
if data, e = operation.GzipData(data); e != nil {
return
}
isGzipped = true
}
if ext == ".gz" {
if strings.HasSuffix(fileName, ".css.gz") ||
strings.HasSuffix(fileName, ".html.gz") ||
strings.HasSuffix(fileName, ".txt.gz") ||
strings.HasSuffix(fileName, ".js.gz") {
fileName = fileName[:len(fileName)-3]
isGzipped = true
}
}
if r.Method == "POST" {
fileName, data, mimeType, isGzipped, e = parseMultipart(r, isChunkedFile)
} else {
isGzipped = false
mimeType = r.Header.Get("Content-Type")
fileName = ""
data, e = ioutil.ReadAll(r.Body)
}
if e != nil {
return
}
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
ttl, _ = ReadTTL(r.FormValue("ttl"))

100
weed/storage/needle_parse_multipart.go Normal file
View file

@ -0,0 +1,100 @@
package storage
import (
"mime"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/glog"
"net/http"
"path"
"io/ioutil"
"strings"
)
func parseMultipart(r *http.Request, isChunkedFile bool) (
fileName string, data []byte, mimeType string, isGzipped bool, e error) {
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
return
}
//first multi-part item
part, fe := form.NextPart()
if fe != nil {
glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
e = fe
return
}
fileName = part.FileName()
if fileName != "" {
fileName = path.Base(fileName)
}
data, e = ioutil.ReadAll(part)
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
return
}
//if the filename is empty string, do a search on the other multi-part items
for fileName == "" {
part2, fe := form.NextPart()
if fe != nil {
break // no more or on error, just safely break
}
fName := part2.FileName()
//found the first <file type> multi-part has filename
if fName != "" {
data2, fe2 := ioutil.ReadAll(part2)
if fe2 != nil {
glog.V(0).Infoln("Reading Content [ERROR]", fe2)
e = fe2
return
}
//update
data = data2
fileName = path.Base(fName)
break
}
}
if !isChunkedFile {
dotIndex := strings.LastIndex(fileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(fileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && mtype != contentType {
mimeType = contentType //only return mime type if not deductable
mtype = contentType
}
if part.Header.Get("Content-Encoding") == "gzip" {
isGzipped = true
} else if operation.IsGzippable(ext, mtype) {
if data, e = operation.GzipData(data); e != nil {
return
}
isGzipped = true
}
if ext == ".gz" {
if strings.HasSuffix(fileName, ".css.gz") ||
strings.HasSuffix(fileName, ".html.gz") ||
strings.HasSuffix(fileName, ".txt.gz") ||
strings.HasSuffix(fileName, ".js.gz") {
fileName = fileName[:len(fileName)-3]
isGzipped = true
}
}
}
return
}