// +build windows package memory_map import ( "os" "reflect" "syscall" "unsafe" "golang.org/x/sys/windows" ) type MemoryBuffer struct { aligned_length uint64 length uint64 aligned_ptr uintptr ptr uintptr Buffer []byte } type MemoryMap struct { File *os.File file_memory_map_handle uintptr write_map_views []MemoryBuffer max_length uint64 End_of_file int64 } var FileMemoryMap = make(map[string]*MemoryMap) type DWORD = uint32 type WORD = uint16 var ( modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize") procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize") ) var currentProcess, _ = windows.GetCurrentProcess() var currentMinWorkingSet uint64 = 0 var currentMaxWorkingSet uint64 = 0 var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet) var systemInfo, _ = getSystemInfo() var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256 func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { chunks := (maxLength / chunkSize) if chunks*chunkSize < maxLength { chunks = chunks + 1 } alignedMaxLength := chunks * chunkSize maxLength_high := uint32(alignedMaxLength >> 32) maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF) file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil) if err == nil { mMap.File = file mMap.file_memory_map_handle = uintptr(file_memory_map_handle) mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize) mMap.max_length = alignedMaxLength mMap.End_of_file = -1 } } func (mMap *MemoryMap) DeleteFileAndMemoryMap() { //First we close the file handles first to delete the file, //Then we unmap the memory to ensure the unmapping process doesn't write the data to disk windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle)) windows.CloseHandle(windows.Handle(mMap.File.Fd())) for _, view := range mMap.write_map_views { view.ReleaseMemory() } mMap.write_map_views = nil mMap.max_length = 0 } func min(x, y uint64) uint64 { if x < y { return x } return y } func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { for { if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) { allocateChunk(mMap) } else { break } } remaining_length := length sliceIndex := offset / chunkSize sliceOffset := offset - (sliceIndex * chunkSize) dataOffset := uint64(0) for { writeEnd := min((remaining_length + sliceOffset), chunkSize) copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:]) remaining_length -= (writeEnd - sliceOffset) dataOffset += (writeEnd - sliceOffset) if remaining_length > 0 { sliceIndex += 1 sliceOffset = 0 } else { break } } if mMap.End_of_file < int64(offset+length-1) { mMap.End_of_file = int64(offset + length - 1) } } func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) { return allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false) } func (mBuffer *MemoryBuffer) ReleaseMemory() { currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) windows.UnmapViewOfFile(mBuffer.aligned_ptr) var _ = setProcessWorkingSetSize(uintptr(currentProcess), uintptr(currentMinWorkingSet), uintptr(currentMaxWorkingSet)) mBuffer.ptr = 0 mBuffer.aligned_ptr = 0 mBuffer.length = 0 mBuffer.aligned_length = 0 mBuffer.Buffer = nil } func allocateChunk(mMap *MemoryMap) { start := uint64(len(mMap.write_map_views)) * chunkSize mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true) if err == nil { mMap.write_map_views = append(mMap.write_map_views, mBuffer) windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) } } func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) { mBuffer := MemoryBuffer{} //align memory allocations to the minium virtal memory allocation size dwSysGran := systemInfo.dwAllocationGranularity start := (offset / uint64(dwSysGran)) * uint64(dwSysGran) diff := offset - start aligned_length := diff + length offset_high := uint32(start >> 32) offset_low := uint32(start & 0xFFFFFFFF) access := windows.FILE_MAP_READ if write { access = windows.FILE_MAP_WRITE } currentMinWorkingSet = currentMinWorkingSet + aligned_length currentMaxWorkingSet = currentMaxWorkingSet + aligned_length // increase the process working set size to hint to windows memory manager to // prioritise keeping this memory mapped in physical memory over other standby memory var _ = setProcessWorkingSetSize(uintptr(currentProcess), uintptr(currentMinWorkingSet), uintptr(currentMaxWorkingSet)) addr_ptr, errno := windows.MapViewOfFile(hMapFile, uint32(access), // read/write permission offset_high, offset_low, uintptr(aligned_length)) if addr_ptr == 0 { return mBuffer, errno } mBuffer.aligned_ptr = addr_ptr mBuffer.aligned_length = aligned_length mBuffer.ptr = addr_ptr + uintptr(diff) mBuffer.length = length slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer)) slice_header.Data = addr_ptr + uintptr(diff) slice_header.Len = int(length) slice_header.Cap = int(length) return mBuffer, nil } // typedef struct _SYSTEM_INFO { // union { // DWORD dwOemId; // struct { // WORD wProcessorArchitecture; // WORD wReserved; // }; // }; // DWORD dwPageSize; // LPVOID lpMinimumApplicationAddress; // LPVOID lpMaximumApplicationAddress; // DWORD_PTR dwActiveProcessorMask; // DWORD dwNumberOfProcessors; // DWORD dwProcessorType; // DWORD dwAllocationGranularity; // WORD wProcessorLevel; // WORD wProcessorRevision; // } SYSTEM_INFO; // https://msdn.microsoft.com/en-us/library/ms724958(v=vs.85).aspx type _SYSTEM_INFO struct { dwOemId DWORD dwPageSize DWORD lpMinimumApplicationAddress uintptr lpMaximumApplicationAddress uintptr dwActiveProcessorMask uintptr dwNumberOfProcessors DWORD dwProcessorType DWORD dwAllocationGranularity DWORD wProcessorLevel WORD wProcessorRevision WORD } // void WINAPI GetSystemInfo( // _Out_ LPSYSTEM_INFO lpSystemInfo // ); // https://msdn.microsoft.com/en-us/library/ms724381(VS.85).aspx func getSystemInfo() (_SYSTEM_INFO, error) { var si _SYSTEM_INFO _, _, err := procGetSystemInfo.Call( uintptr(unsafe.Pointer(&si)), ) if err != syscall.Errno(0) { return si, err } return si, nil } // BOOL GetProcessWorkingSetSize( // HANDLE hProcess, // PSIZE_T lpMinimumWorkingSetSize, // PSIZE_T lpMaximumWorkingSetSize // ); // https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getprocessworkingsetsize func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error { r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet))) if r1 == 0 { if err != syscall.Errno(0) { return err } } return nil } // BOOL SetProcessWorkingSetSize( // HANDLE hProcess, // SIZE_T dwMinimumWorkingSetSize, // SIZE_T dwMaximumWorkingSetSize // ); // https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setprocessworkingsetsize func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uintptr, dwMaxWorkingSet uintptr) error { r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, (dwMinWorkingSet), (dwMaxWorkingSet)) if r1 == 0 { if err != syscall.Errno(0) { return err } } return nil }