2019-10-29 07:35:16 +00:00
|
|
|
package backend
|
|
|
|
|
|
|
|
import (
|
2022-07-29 07:17:28 +00:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
|
|
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
2019-10-29 07:35:16 +00:00
|
|
|
"os"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2019-11-29 02:33:18 +00:00
|
|
|
_ BackendStorageFile = &DiskFile{}
|
2019-10-29 07:35:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type DiskFile struct {
|
|
|
|
File *os.File
|
|
|
|
fullFilePath string
|
2021-02-20 20:42:09 +00:00
|
|
|
fileSize int64
|
|
|
|
modTime time.Time
|
2019-10-29 07:35:16 +00:00
|
|
|
}
|
|
|
|
|
2019-10-30 05:37:36 +00:00
|
|
|
func NewDiskFile(f *os.File) *DiskFile {
|
2021-02-20 20:42:09 +00:00
|
|
|
stat, err := f.Stat()
|
|
|
|
if err != nil {
|
|
|
|
glog.Fatalf("stat file %s: %v", f.Name(), err)
|
|
|
|
}
|
2021-02-21 04:06:06 +00:00
|
|
|
offset := stat.Size()
|
|
|
|
if offset%NeedlePaddingSize != 0 {
|
|
|
|
offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
|
|
|
|
}
|
2021-02-20 20:42:09 +00:00
|
|
|
|
2019-10-29 07:35:16 +00:00
|
|
|
return &DiskFile{
|
2019-10-30 05:37:36 +00:00
|
|
|
fullFilePath: f.Name(),
|
2019-10-29 07:35:16 +00:00
|
|
|
File: f,
|
2021-02-21 04:06:06 +00:00
|
|
|
fileSize: offset,
|
2021-02-20 20:42:09 +00:00
|
|
|
modTime: stat.ModTime(),
|
2019-10-29 07:35:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
|
|
|
|
return df.File.ReadAt(p, off)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
|
2021-02-20 20:42:09 +00:00
|
|
|
n, err = df.File.WriteAt(p, off)
|
|
|
|
if err == nil {
|
|
|
|
waterMark := off + int64(n)
|
|
|
|
if waterMark > df.fileSize {
|
|
|
|
df.fileSize = waterMark
|
|
|
|
df.modTime = time.Now()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return
|
2019-10-29 07:35:16 +00:00
|
|
|
}
|
|
|
|
|
2021-03-05 10:29:38 +00:00
|
|
|
func (df *DiskFile) Write(p []byte) (n int, err error) {
|
2021-02-21 04:06:06 +00:00
|
|
|
return df.WriteAt(p, df.fileSize)
|
|
|
|
}
|
|
|
|
|
2019-10-29 07:35:16 +00:00
|
|
|
func (df *DiskFile) Truncate(off int64) error {
|
2021-02-20 20:42:09 +00:00
|
|
|
err := df.File.Truncate(off)
|
|
|
|
if err == nil {
|
|
|
|
df.fileSize = off
|
|
|
|
df.modTime = time.Now()
|
|
|
|
}
|
|
|
|
return err
|
2019-10-29 07:35:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (df *DiskFile) Close() error {
|
|
|
|
return df.File.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
|
2022-03-10 13:58:56 +00:00
|
|
|
if df.File == nil {
|
|
|
|
err = os.ErrInvalid
|
|
|
|
}
|
|
|
|
return df.fileSize, df.modTime, err
|
2019-10-29 07:35:16 +00:00
|
|
|
}
|
|
|
|
|
2019-12-09 03:44:16 +00:00
|
|
|
func (df *DiskFile) Name() string {
|
2019-10-29 07:35:16 +00:00
|
|
|
return df.fullFilePath
|
|
|
|
}
|
2020-03-20 06:54:52 +00:00
|
|
|
|
|
|
|
func (df *DiskFile) Sync() error {
|
stuck with file.Sync()
starting to get weed/storage () - (master) > go test ./...
I0630 12:19:54 65819 needle_map_metric_test.go:26] FileCount expected 10000 actual 10000
I0630 12:19:54 65819 needle_map_metric_test.go:27] DeletedSize expected 1648 actual 1648
I0630 12:19:54 65819 needle_map_metric_test.go:28] ContentSize expected 10000 actual 10000
I0630 12:19:54 65819 needle_map_metric_test.go:29] DeletedCount expected 1648 actual 1959
I0630 12:19:54 65819 needle_map_metric_test.go:30] MaxFileKey expected 10000 actual 10000
I0630 12:19:54 65819 volume_loading.go:136] loading index /var/folders/jv/0rlhdck17jzgt7b3hcnq63mc0000gp/T/TestCompaction909350722/001/1.idx to memory
I0630 12:20:36 65819 volume_vacuum.go:98] Committing volume 1 vacuuming...
panic: test timed out after 10m0s
goroutine 61 [running]:
testing.(*M).startAlarm.func1()
/usr/local/go/src/testing/testing.go:2029 +0x8e
created by time.goFunc
/usr/local/go/src/time/sleep.go:176 +0x32
goroutine 1 [chan receive, 10 minutes]:
testing.(*T).Run(0xc0000cd520, {0x1c09bde?, 0x5ab798ea5c102?}, 0x1c67bd8)
/usr/local/go/src/testing/testing.go:1487 +0x37a
testing.runTests.func1(0xc0000cd520?)
/usr/local/go/src/testing/testing.go:1839 +0x6e
testing.tRunner(0xc0000cd520, 0xc00039fcd8)
/usr/local/go/src/testing/testing.go:1439 +0x102
testing.runTests(0xc0000dadc0?, {0x25b4460, 0x6, 0x6}, {0x2855108?, 0x40?, 0x25c4b80?})
/usr/local/go/src/testing/testing.go:1837 +0x457
testing.(*M).Run(0xc0000dadc0)
/usr/local/go/src/testing/testing.go:1719 +0x5d9
main.main()
_testmain.go:57 +0x1aa
goroutine 21 [chan receive]:
github.com/chrislusf/seaweedfs/weed/glog.(*loggingT).flushDaemon(0x0?)
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/glog/glog.go:883 +0x6a
created by github.com/chrislusf/seaweedfs/weed/glog.init.0
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/glog/glog.go:410 +0x1bf
goroutine 55 [syscall]:
syscall.syscall(0x44ac0f96?, 0xc00007ad80?, 0xc00098b480?, 0x10588df?)
/usr/local/go/src/runtime/sys_darwin.go:22 +0x4e
syscall.fcntl(0x100000001?, 0xc00098b4b0?, 0x1069a7c?)
/usr/local/go/src/syscall/zsyscall_darwin_amd64.go:319 +0x30
internal/poll.(*FD).Fsync.func1(...)
/usr/local/go/src/internal/poll/fd_fsync_darwin.go:18
internal/poll.ignoringEINTR(...)
/usr/local/go/src/internal/poll/fd_posix.go:74
internal/poll.(*FD).Fsync(0xc00038f1e0?)
/usr/local/go/src/internal/poll/fd_fsync_darwin.go:17 +0xfc
os.(*File).Sync(0xc00012a030)
/usr/local/go/src/os/file_posix.go:168 +0x4e
github.com/chrislusf/seaweedfs/weed/storage/backend.(*DiskFile).Sync(...)
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/backend/disk_file.go:84
github.com/chrislusf/seaweedfs/weed/storage.(*Volume).makeupDiff(0xc0000bb440, {0xc000633a40, 0x52}, {0xc000633aa0, 0x52}, {0xc000633b00, 0x52}, {0xc000633b60, 0x52})
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/volume_vacuum.go:295 +0x12fa
github.com/chrislusf/seaweedfs/weed/storage.(*Volume).CommitCompact(0xc0000bb440)
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/volume_vacuum.go:119 +0x3e8
github.com/chrislusf/seaweedfs/weed/storage.TestCompaction(0xc00025a000)
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/volume_vacuum_test.go:89 +0x305
testing.tRunner(0xc00025a000, 0x1c67bd8)
/usr/local/go/src/testing/testing.go:1439 +0x102
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1486 +0x35f
goroutine 56 [chan receive, 10 minutes]:
github.com/chrislusf/seaweedfs/weed/storage.(*Volume).startWorker.func1()
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/volume_write.go:244 +0x41a
created by github.com/chrislusf/seaweedfs/weed/storage.(*Volume).startWorker
/Users/chrislu/go/src/github.com/chrislusf/seaweedfs/weed/storage/volume_write.go:234 +0x56
FAIL github.com/chrislusf/seaweedfs/weed/storage 600.194s
? github.com/chrislusf/seaweedfs/weed/storage/backend [no test files]
ok github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map (cached)
? github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend [no test files]
ok github.com/chrislusf/seaweedfs/weed/storage/erasure_coding (cached)
? github.com/chrislusf/seaweedfs/weed/storage/idx [no test files]
ok github.com/chrislusf/seaweedfs/weed/storage/needle (cached)
ok github.com/chrislusf/seaweedfs/weed/storage/needle_map (cached)
ok github.com/chrislusf/seaweedfs/weed/storage/super_block (cached)
? github.com/chrislusf/seaweedfs/weed/storage/types [no test files]
? github.com/chrislusf/seaweedfs/weed/storage/volume_info [no test files]
FAIL
2022-06-30 19:32:55 +00:00
|
|
|
return nil
|
|
|
|
// return df.File.Sync()
|
2020-03-20 06:54:52 +00:00
|
|
|
}
|