From d5f5acb734738e89326ca49aee241b1f6a6c270c Mon Sep 17 00:00:00 2001 From: "j.laycock" Date: Fri, 20 Sep 2019 12:44:29 +0100 Subject: [PATCH] limit locking physical memory to 80% of max physical memory --- weed/storage/memory_map/memory_map_windows.go | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/weed/storage/memory_map/memory_map_windows.go b/weed/storage/memory_map/memory_map_windows.go index cf27c629e..a06b7947c 100644 --- a/weed/storage/memory_map/memory_map_windows.go +++ b/weed/storage/memory_map/memory_map_windows.go @@ -29,6 +29,7 @@ type MemoryMap struct { var FileMemoryMap = make(map[string]*MemoryMap) +type DWORDLONG = uint64 type DWORD = uint32 type WORD = uint16 @@ -36,6 +37,7 @@ var ( modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") + procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx") procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize") procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize") ) @@ -48,6 +50,9 @@ var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, var systemInfo, _ = getSystemInfo() var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256 +var memoryStatusEx, _ = globalMemoryStatusEx() +var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8) + func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { chunks := (maxLength / chunkSize) @@ -131,13 +136,15 @@ func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, e func (mBuffer *MemoryBuffer) ReleaseMemory() { - currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length - currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length - windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) windows.UnmapViewOfFile(mBuffer.aligned_ptr) - var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) + currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length + currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length + + if currentMinWorkingSet < maxMemoryLimitBytes { + var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) + } mBuffer.ptr = 0 mBuffer.aligned_ptr = 0 @@ -152,7 +159,6 @@ func allocateChunk(mMap *MemoryMap) { if err == nil { mMap.write_map_views = append(mMap.write_map_views, mBuffer) - windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) } } @@ -179,9 +185,11 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) currentMinWorkingSet = currentMinWorkingSet + aligned_length currentMaxWorkingSet = currentMaxWorkingSet + aligned_length - // increase the process working set size to hint to windows memory manager to - // prioritise keeping this memory mapped in physical memory over other standby memory - var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) + if currentMinWorkingSet < maxMemoryLimitBytes { + // increase the process working set size to hint to windows memory manager to + // prioritise keeping this memory mapped in physical memory over other standby memory + var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) + } addr_ptr, errno := windows.MapViewOfFile(hMapFile, uint32(access), // read/write permission @@ -193,6 +201,10 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) return mBuffer, errno } + if currentMinWorkingSet < maxMemoryLimitBytes { + windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) + } + mBuffer.aligned_ptr = addr_ptr mBuffer.aligned_length = aligned_length mBuffer.ptr = addr_ptr + uintptr(diff) @@ -206,6 +218,47 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) return mBuffer, nil } +//typedef struct _MEMORYSTATUSEX { +// DWORD dwLength; +// DWORD dwMemoryLoad; +// DWORDLONG ullTotalPhys; +// DWORDLONG ullAvailPhys; +// DWORDLONG ullTotalPageFile; +// DWORDLONG ullAvailPageFile; +// DWORDLONG ullTotalVirtual; +// DWORDLONG ullAvailVirtual; +// DWORDLONG ullAvailExtendedVirtual; +// } MEMORYSTATUSEX, *LPMEMORYSTATUSEX; +//https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex + +type _MEMORYSTATUSEX struct { + dwLength DWORD + dwMemoryLoad DWORD + ullTotalPhys DWORDLONG + ullAvailPhys DWORDLONG + ullTotalPageFile DWORDLONG + ullAvailPageFile DWORDLONG + ullTotalVirtual DWORDLONG + ullAvailVirtual DWORDLONG + ullAvailExtendedVirtual DWORDLONG +} + +// BOOL GlobalMemoryStatusEx( +// LPMEMORYSTATUSEX lpBuffer +// ); +// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex +func globalMemoryStatusEx() (_MEMORYSTATUSEX, error) { + var mem_status _MEMORYSTATUSEX + + mem_status.dwLength = uint32(unsafe.Sizeof(mem_status)) + _, _, err := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&mem_status))) + + if err != syscall.Errno(0) { + return mem_status, err + } + return mem_status, nil +} + // typedef struct _SYSTEM_INFO { // union { // DWORD dwOemId; @@ -224,7 +277,7 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) // WORD wProcessorLevel; // WORD wProcessorRevision; // } SYSTEM_INFO; -// https://msdn.microsoft.com/en-us/library/ms724958(v=vs.85).aspx +// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info type _SYSTEM_INFO struct { dwOemId DWORD dwPageSize DWORD @@ -241,12 +294,10 @@ type _SYSTEM_INFO struct { // void WINAPI GetSystemInfo( // _Out_ LPSYSTEM_INFO lpSystemInfo // ); -// https://msdn.microsoft.com/en-us/library/ms724381(VS.85).aspx +// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsysteminfo func getSystemInfo() (_SYSTEM_INFO, error) { var si _SYSTEM_INFO - _, _, err := procGetSystemInfo.Call( - uintptr(unsafe.Pointer(&si)), - ) + _, _, err := procGetSystemInfo.Call(uintptr(unsafe.Pointer(&si))) if err != syscall.Errno(0) { return si, err }