mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Make releaseMemory private and return byte array instead, fix other platform compilation issues, reduce in-memory chunk size.
This commit is contained in:
parent
d5f5acb734
commit
eb27c2b037
|
@ -25,16 +25,16 @@ type MemoryMap struct {
|
||||||
|
|
||||||
var FileMemoryMap = make(map[string]*MemoryMap)
|
var FileMemoryMap = make(map[string]*MemoryMap)
|
||||||
|
|
||||||
|
func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) {
|
||||||
|
}
|
||||||
|
|
||||||
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
|
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
|
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) ([]byte, error) {
|
||||||
return MemoryBuffer{}, fmt.Errorf("Memory Map not implemented for this platform")
|
dataSlice := []byte{}
|
||||||
}
|
return dataSlice, fmt.Errorf("Memory Map not implemented for this platform")
|
||||||
|
|
||||||
func (mBuffer *MemoryBuffer) ReleaseMemory() {
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() {
|
func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() {
|
||||||
|
|
|
@ -48,7 +48,7 @@ var currentMaxWorkingSet uint64 = 0
|
||||||
var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet)
|
var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet)
|
||||||
|
|
||||||
var systemInfo, _ = getSystemInfo()
|
var systemInfo, _ = getSystemInfo()
|
||||||
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256
|
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 128
|
||||||
|
|
||||||
var memoryStatusEx, _ = globalMemoryStatusEx()
|
var memoryStatusEx, _ = globalMemoryStatusEx()
|
||||||
var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8)
|
var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8)
|
||||||
|
@ -82,7 +82,7 @@ func (mMap *MemoryMap) DeleteFileAndMemoryMap() {
|
||||||
windows.CloseHandle(windows.Handle(mMap.File.Fd()))
|
windows.CloseHandle(windows.Handle(mMap.File.Fd()))
|
||||||
|
|
||||||
for _, view := range mMap.write_map_views {
|
for _, view := range mMap.write_map_views {
|
||||||
view.ReleaseMemory()
|
view.releaseMemory()
|
||||||
}
|
}
|
||||||
|
|
||||||
mMap.write_map_views = nil
|
mMap.write_map_views = nil
|
||||||
|
@ -130,17 +130,21 @@ func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
|
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (dataSlice []byte, err error) {
|
||||||
return allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false)
|
dataSlice = make([]byte, length)
|
||||||
|
mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false)
|
||||||
|
copy(dataSlice, mBuffer.Buffer)
|
||||||
|
mBuffer.releaseMemory()
|
||||||
|
return dataSlice, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mBuffer *MemoryBuffer) ReleaseMemory() {
|
func (mBuffer *MemoryBuffer) releaseMemory() {
|
||||||
|
|
||||||
windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
|
windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
|
||||||
windows.UnmapViewOfFile(mBuffer.aligned_ptr)
|
windows.UnmapViewOfFile(mBuffer.aligned_ptr)
|
||||||
|
|
||||||
currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length
|
currentMinWorkingSet -= mBuffer.aligned_length
|
||||||
currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length
|
currentMaxWorkingSet -= mBuffer.aligned_length
|
||||||
|
|
||||||
if currentMinWorkingSet < maxMemoryLimitBytes {
|
if currentMinWorkingSet < maxMemoryLimitBytes {
|
||||||
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
|
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
|
||||||
|
@ -182,8 +186,8 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool)
|
||||||
access = windows.FILE_MAP_WRITE
|
access = windows.FILE_MAP_WRITE
|
||||||
}
|
}
|
||||||
|
|
||||||
currentMinWorkingSet = currentMinWorkingSet + aligned_length
|
currentMinWorkingSet += aligned_length
|
||||||
currentMaxWorkingSet = currentMaxWorkingSet + aligned_length
|
currentMaxWorkingSet += aligned_length
|
||||||
|
|
||||||
if currentMinWorkingSet < maxMemoryLimitBytes {
|
if currentMinWorkingSet < maxMemoryLimitBytes {
|
||||||
// increase the process working set size to hint to windows memory manager to
|
// increase the process working set size to hint to windows memory manager to
|
||||||
|
|
|
@ -164,13 +164,11 @@ func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32
|
||||||
func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dataSlice []byte, err error) {
|
func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dataSlice []byte, err error) {
|
||||||
|
|
||||||
dataSize := GetActualSize(size, version)
|
dataSize := GetActualSize(size, version)
|
||||||
dataSlice = make([]byte, dataSize)
|
dataSlice = make([]byte, int(dataSize))
|
||||||
|
|
||||||
mMap, exists := memory_map.FileMemoryMap[r.Name()]
|
mMap, exists := memory_map.FileMemoryMap[r.Name()]
|
||||||
if exists {
|
if exists {
|
||||||
mBuffer, err := mMap.ReadMemory(uint64(offset), uint64(dataSize))
|
dataSlice, err := mMap.ReadMemory(uint64(offset), uint64(dataSize))
|
||||||
copy(dataSlice, mBuffer.Buffer)
|
|
||||||
mBuffer.ReleaseMemory()
|
|
||||||
return dataSlice, err
|
return dataSlice, err
|
||||||
} else {
|
} else {
|
||||||
_, err = r.ReadAt(dataSlice, offset)
|
_, err = r.ReadAt(dataSlice, offset)
|
||||||
|
@ -291,10 +289,7 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, byt
|
||||||
|
|
||||||
mMap, exists := memory_map.FileMemoryMap[r.Name()]
|
mMap, exists := memory_map.FileMemoryMap[r.Name()]
|
||||||
if exists {
|
if exists {
|
||||||
mBuffer, err := mMap.ReadMemory(uint64(offset), NeedleHeaderSize)
|
bytes, err = mMap.ReadMemory(uint64(offset), NeedleHeaderSize)
|
||||||
copy(bytes, mBuffer.Buffer)
|
|
||||||
mBuffer.ReleaseMemory()
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, bytes, 0, err
|
return nil, bytes, 0, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -113,13 +113,11 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
|
||||||
header := make([]byte, _SuperBlockSize)
|
header := make([]byte, _SuperBlockSize)
|
||||||
mMap, exists := memory_map.FileMemoryMap[dataFile.Name()]
|
mMap, exists := memory_map.FileMemoryMap[dataFile.Name()]
|
||||||
if exists {
|
if exists {
|
||||||
mBuffer, e := mMap.ReadMemory(0, _SuperBlockSize)
|
header, err = mMap.ReadMemory(0, _SuperBlockSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e)
|
err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
copy(header, mBuffer.Buffer)
|
|
||||||
mBuffer.ReleaseMemory()
|
|
||||||
} else {
|
} else {
|
||||||
if _, err = dataFile.Seek(0, 0); err != nil {
|
if _, err = dataFile.Seek(0, 0); err != nil {
|
||||||
err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err)
|
err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err)
|
||||||
|
|
Loading…
Reference in a new issue