gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Refactor to pass memory maps by reference instead of value, fix memory maps not being created properly or written to properly

Browse source
This commit is contained in:
j.laycock 2019-08-30 12:15:17 +01:00
parent dc50701e7d
commit 840ccdc35d
5 changed files with 39 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
weed/storage/memory_map/memory_map_windows.go
View file

@ -27,7 +27,7 @@ type MemoryMap struct {
End_Of_File int64
}
var FileMemoryMap = make(map[string]MemoryMap)
var FileMemoryMap = make(map[string]*MemoryMap)
type DWORD = uint32
type WORD = uint16
@ -40,29 +40,27 @@ var system_info, err = getSystemInfo()
var chunk_size = uint64(system_info.dwAllocationGranularity) * 512
func CreateMemoryMap(file *os.File, maxlength uint64) MemoryMap {
func (mem_map *MemoryMap) CreateMemoryMap(file *os.File, maxlength uint64) {
mem_map := MemoryMap{}
maxlength_high := uint32(maxlength >> 32)
maxlength_low := uint32(maxlength & 0xFFFFFFFF)
file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxlength_high, maxlength_low, nil)
if err != nil {
if err == nil {
mem_map.File = file
mem_map.file_memory_map_handle = uintptr(file_memory_map_handle)
mem_map.write_map_views = make([]MemoryBuffer, 0, maxlength/chunk_size)
mem_map.max_length = maxlength
mem_map.End_Of_File = -1
}
return mem_map
}
func DeleteFileAndMemoryMap(mem_map MemoryMap) {
func (mem_map *MemoryMap) DeleteFileAndMemoryMap() {
windows.CloseHandle(windows.Handle(mem_map.file_memory_map_handle))
windows.CloseHandle(windows.Handle(mem_map.File.Fd()))
for _, view := range mem_map.write_map_views {
ReleaseMemory(view)
view.ReleaseMemory()
}
mem_map.write_map_views = nil
@ -76,7 +74,7 @@ func min(x, y uint64) uint64 {
return y
}
func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
func (mem_map *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
for {
if ((offset+length)/chunk_size)+1 > uint64(len(mem_map.write_map_views)) {
@ -92,7 +90,7 @@ func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
data_offset := uint64(0)
for {
write_end := min(remaining_length, chunk_size)
write_end := min((remaining_length + slice_offset), chunk_size)
copy(mem_map.write_map_views[slice_index].Buffer[slice_offset:write_end], data[data_offset:])
remaining_length -= (write_end - slice_offset)
data_offset += (write_end - slice_offset)
@ -105,16 +103,16 @@ func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
}
}
if mem_map.End_Of_File < int64(offset+length) {
mem_map.End_Of_File = int64(offset + length)
if mem_map.End_Of_File < int64(offset+length-1) {
mem_map.End_Of_File = int64(offset + length - 1)
}
}
func ReadMemory(mem_map MemoryMap, offset uint64, length uint64) (MemoryBuffer, error) {
func (mem_map *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
return allocate(windows.Handle(mem_map.file_memory_map_handle), offset, length, false)
}
func ReleaseMemory(mem_buffer MemoryBuffer) {
func (mem_buffer *MemoryBuffer) ReleaseMemory() {
windows.UnmapViewOfFile(mem_buffer.aligned_ptr)
mem_buffer.ptr = 0
@ -124,9 +122,9 @@ func ReleaseMemory(mem_buffer MemoryBuffer) {
mem_buffer.Buffer = nil
}
func allocateChunk(mem_map MemoryMap) {
func allocateChunk(mem_map *MemoryMap) {
start := uint64(len(mem_map.write_map_views)-1) * chunk_size
start := uint64(len(mem_map.write_map_views)) * chunk_size
mem_buffer, err := allocate(windows.Handle(mem_map.file_memory_map_handle), start, chunk_size, true)
if err == nil {

16
weed/storage/needle/needle_read_write.go
View file

@ -8,10 +8,10 @@ import (
"math"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
. "github.com/joeslay/seaweedfs/weed/storage/types"
"github.com/joeslay/seaweedfs/weed/util"
)
const (
@ -152,7 +152,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32
if err == nil {
if exists {
memory_map.WriteMemory(mem_map, offset, uint64(len(bytesToWrite)), bytesToWrite)
mem_map.WriteMemory(offset, uint64(len(bytesToWrite)), bytesToWrite)
} else {
_, err = w.Write(bytesToWrite)
}
@ -168,9 +168,9 @@ func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dat
mem_map, exists := memory_map.FileMemoryMap[r.Name()]
if exists {
mem_buffer, err := memory_map.ReadMemory(mem_map, uint64(offset), uint64(dataSize))
mem_buffer, err := mem_map.ReadMemory(uint64(offset), uint64(dataSize))
copy(dataSlice, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
return dataSlice, err
} else {
_, err = r.ReadAt(dataSlice, offset)
@ -291,9 +291,9 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, byt
mem_map, exists := memory_map.FileMemoryMap[r.Name()]
if exists {
mem_buffer, err := memory_map.ReadMemory(mem_map, uint64(offset), NeedleHeaderSize)
mem_buffer, err := mem_map.ReadMemory(uint64(offset), NeedleHeaderSize)
copy(bytes, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
if err != nil {
return nil, bytes, 0, err

8
weed/storage/volume_create_windows.go
View file

@ -8,7 +8,7 @@ import (
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"golang.org/x/sys/windows"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/os_overloads"
)
@ -17,8 +17,10 @@ func createVolumeFile(fileName string, preallocate int64) (*os.File, error) {
mem_map, exists := memory_map.FileMemoryMap[fileName]
if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true)
new_mem_map := memory_map.CreateMemoryMap(file, 2^32)
memory_map.FileMemoryMap[fileName] = new_mem_map
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mem_map := memory_map.FileMemoryMap[fileName]
new_mem_map.CreateMemoryMap(file, 1024*1024*1024*4)
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

8
weed/storage/volume_read_write.go
View file

@ -8,10 +8,10 @@ import (
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"github.com/joeslay/seaweedfs/weed/storage/needle"
. "github.com/joeslay/seaweedfs/weed/storage/types"
)
var ErrorNotFound = errors.New("not found")
@ -51,7 +51,7 @@ func (v *Volume) Destroy() (err error) {
}
mem_map, exists := memory_map.FileMemoryMap[v.FileName()]
if exists {
memory_map.DeleteFileAndMemoryMap(mem_map)
mem_map.DeleteFileAndMemoryMap()
delete(memory_map.FileMemoryMap, v.FileName())
}

16
weed/storage/volume_super_block.go
View file

@ -6,11 +6,11 @@ import (
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/pb/master_pb"
"github.com/joeslay/seaweedfs/weed/storage/needle"
"github.com/joeslay/seaweedfs/weed/util"
)
const (
@ -73,11 +73,11 @@ func (s *SuperBlock) Bytes() []byte {
func (v *Volume) maybeWriteSuperBlock() error {
mem_map, exists := memory_map.FileMemoryMap[v.FileName()]
mem_map, exists := memory_map.FileMemoryMap[v.dataFile.Name()]
if exists {
if mem_map.End_Of_File == -1 {
v.SuperBlock.version = needle.CurrentVersion
memory_map.WriteMemory(mem_map, 0, uint64(len(v.SuperBlock.Bytes())), v.SuperBlock.Bytes())
mem_map.WriteMemory(0, uint64(len(v.SuperBlock.Bytes())), v.SuperBlock.Bytes())
}
return nil
} else {
@ -113,13 +113,13 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
header := make([]byte, _SuperBlockSize)
mem_map, exists := memory_map.FileMemoryMap[dataFile.Name()]
if exists {
mem_buffer, e := memory_map.ReadMemory(mem_map, 0, _SuperBlockSize)
mem_buffer, e := mem_map.ReadMemory(0, _SuperBlockSize)
if err != nil {
err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e)
return
}
copy(header, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
} else {
if _, err = dataFile.Seek(0, 0); err != nil {
err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err)