Browse Source
Merge pull request #1087 from joeslay/master
Merge pull request #1087 from joeslay/master
Storing files in in-memory collections on windowspull/1091/head
Chris Lu
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1031 additions and 230 deletions
-
4go.mod
-
3go.sum
-
4weed/command/backup.go
-
2weed/command/compact.go
-
168weed/os_overloads/file_windows.go
-
80weed/os_overloads/syscall_windows.go
-
9weed/os_overloads/types_windows.go
-
1weed/pb/master.proto
-
33weed/pb/master_pb/master.pb.go
-
1weed/pb/volume_server.proto
-
30weed/pb/volume_server_pb/volume_server.pb.go
-
15weed/server/master_grpc_server_volume.go
-
20weed/server/master_server_handlers_admin.go
-
1weed/server/volume_grpc_admin.go
-
2weed/storage/disk_location.go
-
42weed/storage/memory_map/memory_map.go
-
343weed/storage/memory_map/memory_map_windows.go
-
160weed/storage/needle/needle_read_write.go
-
14weed/storage/needle/volume_memory_map_max_size.go
-
10weed/storage/needle/volume_memory_map_max_size_test.go
-
8weed/storage/store.go
-
19weed/storage/volume.go
-
6weed/storage/volume_create.go
-
4weed/storage/volume_create_linux.go
-
38weed/storage/volume_create_windows.go
-
2weed/storage/volume_loading.go
-
7weed/storage/volume_read_write.go
-
67weed/storage/volume_super_block.go
-
138weed/storage/volume_vacuum.go
-
4weed/storage/volume_vacuum_test.go
-
11weed/topology/allocate_volume.go
-
15weed/topology/volume_growth.go
@ -0,0 +1,168 @@ |
|||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
|
// Use of this source code is governed by a BSD-style
|
||||
|
// license that can be found in the LICENSE file.
|
||||
|
|
||||
|
package os_overloads |
||||
|
|
||||
|
import ( |
||||
|
"os" |
||||
|
"syscall" |
||||
|
|
||||
|
"golang.org/x/sys/windows" |
||||
|
) |
||||
|
|
||||
|
func isAbs(path string) (b bool) { |
||||
|
v := volumeName(path) |
||||
|
if v == "" { |
||||
|
return false |
||||
|
} |
||||
|
path = path[len(v):] |
||||
|
if path == "" { |
||||
|
return false |
||||
|
} |
||||
|
return os.IsPathSeparator(path[0]) |
||||
|
} |
||||
|
|
||||
|
func volumeName(path string) (v string) { |
||||
|
if len(path) < 2 { |
||||
|
return "" |
||||
|
} |
||||
|
// with drive letter
|
||||
|
c := path[0] |
||||
|
if path[1] == ':' && |
||||
|
('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || |
||||
|
'A' <= c && c <= 'Z') { |
||||
|
return path[:2] |
||||
|
} |
||||
|
// is it UNC
|
||||
|
if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) && |
||||
|
!os.IsPathSeparator(path[2]) && path[2] != '.' { |
||||
|
// first, leading `\\` and next shouldn't be `\`. its server name.
|
||||
|
for n := 3; n < l-1; n++ { |
||||
|
// second, next '\' shouldn't be repeated.
|
||||
|
if os.IsPathSeparator(path[n]) { |
||||
|
n++ |
||||
|
// third, following something characters. its share name.
|
||||
|
if !os.IsPathSeparator(path[n]) { |
||||
|
if path[n] == '.' { |
||||
|
break |
||||
|
} |
||||
|
for ; n < l; n++ { |
||||
|
if os.IsPathSeparator(path[n]) { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
return path[:n] |
||||
|
} |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return "" |
||||
|
} |
||||
|
|
||||
|
// fixLongPath returns the extended-length (\\?\-prefixed) form of
|
||||
|
// path when needed, in order to avoid the default 260 character file
|
||||
|
// path limit imposed by Windows. If path is not easily converted to
|
||||
|
// the extended-length form (for example, if path is a relative path
|
||||
|
// or contains .. elements), or is short enough, fixLongPath returns
|
||||
|
// path unmodified.
|
||||
|
//
|
||||
|
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
|
||||
|
func fixLongPath(path string) string { |
||||
|
// Do nothing (and don't allocate) if the path is "short".
|
||||
|
// Empirically (at least on the Windows Server 2013 builder),
|
||||
|
// the kernel is arbitrarily okay with < 248 bytes. That
|
||||
|
// matches what the docs above say:
|
||||
|
// "When using an API to create a directory, the specified
|
||||
|
// path cannot be so long that you cannot append an 8.3 file
|
||||
|
// name (that is, the directory name cannot exceed MAX_PATH
|
||||
|
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
|
||||
|
//
|
||||
|
// The MSDN docs appear to say that a normal path that is 248 bytes long
|
||||
|
// will work; empirically the path must be less then 248 bytes long.
|
||||
|
if len(path) < 248 { |
||||
|
// Don't fix. (This is how Go 1.7 and earlier worked,
|
||||
|
// not automatically generating the \\?\ form)
|
||||
|
return path |
||||
|
} |
||||
|
|
||||
|
// The extended form begins with \\?\, as in
|
||||
|
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
|
||||
|
// The extended form disables evaluation of . and .. path
|
||||
|
// elements and disables the interpretation of / as equivalent
|
||||
|
// to \. The conversion here rewrites / to \ and elides
|
||||
|
// . elements as well as trailing or duplicate separators. For
|
||||
|
// simplicity it avoids the conversion entirely for relative
|
||||
|
// paths or paths containing .. elements. For now,
|
||||
|
// \\server\share paths are not converted to
|
||||
|
// \\?\UNC\server\share paths because the rules for doing so
|
||||
|
// are less well-specified.
|
||||
|
if len(path) >= 2 && path[:2] == `\\` { |
||||
|
// Don't canonicalize UNC paths.
|
||||
|
return path |
||||
|
} |
||||
|
if !isAbs(path) { |
||||
|
// Relative path
|
||||
|
return path |
||||
|
} |
||||
|
|
||||
|
const prefix = `\\?` |
||||
|
|
||||
|
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) |
||||
|
copy(pathbuf, prefix) |
||||
|
n := len(path) |
||||
|
r, w := 0, len(prefix) |
||||
|
for r < n { |
||||
|
switch { |
||||
|
case os.IsPathSeparator(path[r]): |
||||
|
// empty block
|
||||
|
r++ |
||||
|
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): |
||||
|
// /./
|
||||
|
r++ |
||||
|
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): |
||||
|
// /../ is currently unhandled
|
||||
|
return path |
||||
|
default: |
||||
|
pathbuf[w] = '\\' |
||||
|
w++ |
||||
|
for ; r < n && !os.IsPathSeparator(path[r]); r++ { |
||||
|
pathbuf[w] = path[r] |
||||
|
w++ |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
// A drive's root directory needs a trailing \
|
||||
|
if w == len(`\\?\c:`) { |
||||
|
pathbuf[w] = '\\' |
||||
|
w++ |
||||
|
} |
||||
|
return string(pathbuf[:w]) |
||||
|
} |
||||
|
|
||||
|
// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
|
||||
|
func syscallMode(i os.FileMode) (o uint32) { |
||||
|
o |= uint32(i.Perm()) |
||||
|
if i&os.ModeSetuid != 0 { |
||||
|
o |= syscall.S_ISUID |
||||
|
} |
||||
|
if i&os.ModeSetgid != 0 { |
||||
|
o |= syscall.S_ISGID |
||||
|
} |
||||
|
if i&os.ModeSticky != 0 { |
||||
|
o |= syscall.S_ISVTX |
||||
|
} |
||||
|
// No mapping for Go's ModeTemporary (plan9 only).
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
//If the bool is set to true then the file is opened with the parameters FILE_ATTRIBUTE_TEMPORARY and
|
||||
|
// FILE_FLAG_DELETE_ON_CLOSE
|
||||
|
func OpenFile(name string, flag int, perm os.FileMode, setToTempAndDelete bool) (file *os.File, err error) { |
||||
|
r, e := Open(fixLongPath(name), flag|windows.O_CLOEXEC, syscallMode(perm), setToTempAndDelete) |
||||
|
if e != nil { |
||||
|
return nil, e |
||||
|
} |
||||
|
return os.NewFile(uintptr(r), name), nil |
||||
|
} |
||||
@ -0,0 +1,80 @@ |
|||||
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
|
// Use of this source code is governed by a BSD-style
|
||||
|
// license that can be found in the LICENSE file.
|
||||
|
|
||||
|
// Windows system calls.
|
||||
|
|
||||
|
package os_overloads |
||||
|
|
||||
|
import ( |
||||
|
"syscall" |
||||
|
"unsafe" |
||||
|
|
||||
|
"golang.org/x/sys/windows" |
||||
|
) |
||||
|
|
||||
|
// windows api calls
|
||||
|
|
||||
|
//sys CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) [failretval==InvalidHandle] = CreateFileW
|
||||
|
|
||||
|
func makeInheritSa() *syscall.SecurityAttributes { |
||||
|
var sa syscall.SecurityAttributes |
||||
|
sa.Length = uint32(unsafe.Sizeof(sa)) |
||||
|
sa.InheritHandle = 1 |
||||
|
return &sa |
||||
|
} |
||||
|
|
||||
|
// opens the
|
||||
|
func Open(path string, mode int, perm uint32, setToTempAndDelete bool) (fd syscall.Handle, err error) { |
||||
|
if len(path) == 0 { |
||||
|
return syscall.InvalidHandle, windows.ERROR_FILE_NOT_FOUND |
||||
|
} |
||||
|
pathp, err := syscall.UTF16PtrFromString(path) |
||||
|
if err != nil { |
||||
|
return syscall.InvalidHandle, err |
||||
|
} |
||||
|
var access uint32 |
||||
|
switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) { |
||||
|
case windows.O_RDONLY: |
||||
|
access = windows.GENERIC_READ |
||||
|
case windows.O_WRONLY: |
||||
|
access = windows.GENERIC_WRITE |
||||
|
case windows.O_RDWR: |
||||
|
access = windows.GENERIC_READ | windows.GENERIC_WRITE |
||||
|
} |
||||
|
if mode&windows.O_CREAT != 0 { |
||||
|
access |= windows.GENERIC_WRITE |
||||
|
} |
||||
|
if mode&windows.O_APPEND != 0 { |
||||
|
access &^= windows.GENERIC_WRITE |
||||
|
access |= windows.FILE_APPEND_DATA |
||||
|
} |
||||
|
sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE) |
||||
|
var sa *syscall.SecurityAttributes |
||||
|
if mode&windows.O_CLOEXEC == 0 { |
||||
|
sa = makeInheritSa() |
||||
|
} |
||||
|
var createmode uint32 |
||||
|
switch { |
||||
|
case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL): |
||||
|
createmode = windows.CREATE_NEW |
||||
|
case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC): |
||||
|
createmode = windows.CREATE_ALWAYS |
||||
|
case mode&windows.O_CREAT == windows.O_CREAT: |
||||
|
createmode = windows.OPEN_ALWAYS |
||||
|
case mode&windows.O_TRUNC == windows.O_TRUNC: |
||||
|
createmode = windows.TRUNCATE_EXISTING |
||||
|
default: |
||||
|
createmode = windows.OPEN_EXISTING |
||||
|
} |
||||
|
|
||||
|
var h syscall.Handle |
||||
|
var e error |
||||
|
|
||||
|
if setToTempAndDelete { |
||||
|
h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, (windows.FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE), 0) |
||||
|
} else { |
||||
|
h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, windows.FILE_ATTRIBUTE_NORMAL, 0) |
||||
|
} |
||||
|
return h, e |
||||
|
} |
||||
@ -0,0 +1,9 @@ |
|||||
|
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
|
// Use of this source code is governed by a BSD-style
|
||||
|
// license that can be found in the LICENSE file.
|
||||
|
|
||||
|
package os_overloads |
||||
|
|
||||
|
const ( |
||||
|
FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 |
||||
|
) |
||||
@ -0,0 +1,42 @@ |
|||||
|
// +build !windows
|
||||
|
|
||||
|
package memory_map |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"os" |
||||
|
) |
||||
|
|
||||
|
type MemoryBuffer struct { |
||||
|
aligned_length uint64 |
||||
|
length uint64 |
||||
|
aligned_ptr uintptr |
||||
|
ptr uintptr |
||||
|
Buffer []byte |
||||
|
} |
||||
|
|
||||
|
type MemoryMap struct { |
||||
|
File *os.File |
||||
|
file_memory_map_handle uintptr |
||||
|
write_map_views []MemoryBuffer |
||||
|
max_length uint64 |
||||
|
End_of_file int64 |
||||
|
} |
||||
|
|
||||
|
var FileMemoryMap = make(map[string]*MemoryMap) |
||||
|
|
||||
|
func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { |
||||
|
} |
||||
|
|
||||
|
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) ([]byte, error) { |
||||
|
dataSlice := []byte{} |
||||
|
return dataSlice, fmt.Errorf("Memory Map not implemented for this platform") |
||||
|
} |
||||
|
|
||||
|
func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() { |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,343 @@ |
|||||
|
// +build windows
|
||||
|
|
||||
|
package memory_map |
||||
|
|
||||
|
import ( |
||||
|
"os" |
||||
|
"reflect" |
||||
|
"syscall" |
||||
|
"unsafe" |
||||
|
|
||||
|
"golang.org/x/sys/windows" |
||||
|
) |
||||
|
|
||||
|
type MemoryBuffer struct { |
||||
|
aligned_length uint64 |
||||
|
length uint64 |
||||
|
aligned_ptr uintptr |
||||
|
ptr uintptr |
||||
|
Buffer []byte |
||||
|
} |
||||
|
|
||||
|
type MemoryMap struct { |
||||
|
File *os.File |
||||
|
file_memory_map_handle uintptr |
||||
|
write_map_views []MemoryBuffer |
||||
|
max_length uint64 |
||||
|
End_of_file int64 |
||||
|
} |
||||
|
|
||||
|
var FileMemoryMap = make(map[string]*MemoryMap) |
||||
|
|
||||
|
type DWORDLONG = uint64 |
||||
|
type DWORD = uint32 |
||||
|
type WORD = uint16 |
||||
|
|
||||
|
var ( |
||||
|
modkernel32 = syscall.NewLazyDLL("kernel32.dll") |
||||
|
|
||||
|
procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") |
||||
|
procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx") |
||||
|
procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize") |
||||
|
procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize") |
||||
|
) |
||||
|
|
||||
|
var currentProcess, _ = windows.GetCurrentProcess() |
||||
|
var currentMinWorkingSet uint64 = 0 |
||||
|
var currentMaxWorkingSet uint64 = 0 |
||||
|
var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet) |
||||
|
|
||||
|
var systemInfo, _ = getSystemInfo() |
||||
|
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 128 |
||||
|
|
||||
|
var memoryStatusEx, _ = globalMemoryStatusEx() |
||||
|
var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8) |
||||
|
|
||||
|
func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { |
||||
|
|
||||
|
chunks := (maxLength / chunkSize) |
||||
|
if chunks*chunkSize < maxLength { |
||||
|
chunks = chunks + 1 |
||||
|
} |
||||
|
|
||||
|
alignedMaxLength := chunks * chunkSize |
||||
|
|
||||
|
maxLength_high := uint32(alignedMaxLength >> 32) |
||||
|
maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF) |
||||
|
file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil) |
||||
|
|
||||
|
if err == nil { |
||||
|
mMap.File = file |
||||
|
mMap.file_memory_map_handle = uintptr(file_memory_map_handle) |
||||
|
mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize) |
||||
|
mMap.max_length = alignedMaxLength |
||||
|
mMap.End_of_file = -1 |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (mMap *MemoryMap) DeleteFileAndMemoryMap() { |
||||
|
//First we close the file handles first to delete the file,
|
||||
|
//Then we unmap the memory to ensure the unmapping process doesn't write the data to disk
|
||||
|
windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle)) |
||||
|
windows.CloseHandle(windows.Handle(mMap.File.Fd())) |
||||
|
|
||||
|
for _, view := range mMap.write_map_views { |
||||
|
view.releaseMemory() |
||||
|
} |
||||
|
|
||||
|
mMap.write_map_views = nil |
||||
|
mMap.max_length = 0 |
||||
|
} |
||||
|
|
||||
|
func min(x, y uint64) uint64 { |
||||
|
if x < y { |
||||
|
return x |
||||
|
} |
||||
|
return y |
||||
|
} |
||||
|
|
||||
|
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { |
||||
|
|
||||
|
for { |
||||
|
if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) { |
||||
|
allocateChunk(mMap) |
||||
|
} else { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
remaining_length := length |
||||
|
sliceIndex := offset / chunkSize |
||||
|
sliceOffset := offset - (sliceIndex * chunkSize) |
||||
|
dataOffset := uint64(0) |
||||
|
|
||||
|
for { |
||||
|
writeEnd := min((remaining_length + sliceOffset), chunkSize) |
||||
|
copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:]) |
||||
|
remaining_length -= (writeEnd - sliceOffset) |
||||
|
dataOffset += (writeEnd - sliceOffset) |
||||
|
|
||||
|
if remaining_length > 0 { |
||||
|
sliceIndex += 1 |
||||
|
sliceOffset = 0 |
||||
|
} else { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if mMap.End_of_file < int64(offset+length-1) { |
||||
|
mMap.End_of_file = int64(offset + length - 1) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (dataSlice []byte, err error) { |
||||
|
dataSlice = make([]byte, length) |
||||
|
mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false) |
||||
|
copy(dataSlice, mBuffer.Buffer) |
||||
|
mBuffer.releaseMemory() |
||||
|
return dataSlice, err |
||||
|
} |
||||
|
|
||||
|
func (mBuffer *MemoryBuffer) releaseMemory() { |
||||
|
|
||||
|
windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) |
||||
|
windows.UnmapViewOfFile(mBuffer.aligned_ptr) |
||||
|
|
||||
|
currentMinWorkingSet -= mBuffer.aligned_length |
||||
|
currentMaxWorkingSet -= mBuffer.aligned_length |
||||
|
|
||||
|
if currentMinWorkingSet < maxMemoryLimitBytes { |
||||
|
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) |
||||
|
} |
||||
|
|
||||
|
mBuffer.ptr = 0 |
||||
|
mBuffer.aligned_ptr = 0 |
||||
|
mBuffer.length = 0 |
||||
|
mBuffer.aligned_length = 0 |
||||
|
mBuffer.Buffer = nil |
||||
|
} |
||||
|
|
||||
|
func allocateChunk(mMap *MemoryMap) { |
||||
|
start := uint64(len(mMap.write_map_views)) * chunkSize |
||||
|
mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true) |
||||
|
|
||||
|
if err == nil { |
||||
|
mMap.write_map_views = append(mMap.write_map_views, mBuffer) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) { |
||||
|
|
||||
|
mBuffer := MemoryBuffer{} |
||||
|
|
||||
|
//align memory allocations to the minium virtal memory allocation size
|
||||
|
dwSysGran := systemInfo.dwAllocationGranularity |
||||
|
|
||||
|
start := (offset / uint64(dwSysGran)) * uint64(dwSysGran) |
||||
|
diff := offset - start |
||||
|
aligned_length := diff + length |
||||
|
|
||||
|
offset_high := uint32(start >> 32) |
||||
|
offset_low := uint32(start & 0xFFFFFFFF) |
||||
|
|
||||
|
access := windows.FILE_MAP_READ |
||||
|
|
||||
|
if write { |
||||
|
access = windows.FILE_MAP_WRITE |
||||
|
} |
||||
|
|
||||
|
currentMinWorkingSet += aligned_length |
||||
|
currentMaxWorkingSet += aligned_length |
||||
|
|
||||
|
if currentMinWorkingSet < maxMemoryLimitBytes { |
||||
|
// increase the process working set size to hint to windows memory manager to
|
||||
|
// prioritise keeping this memory mapped in physical memory over other standby memory
|
||||
|
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) |
||||
|
} |
||||
|
|
||||
|
addr_ptr, errno := windows.MapViewOfFile(hMapFile, |
||||
|
uint32(access), // read/write permission
|
||||
|
offset_high, |
||||
|
offset_low, |
||||
|
uintptr(aligned_length)) |
||||
|
|
||||
|
if addr_ptr == 0 { |
||||
|
return mBuffer, errno |
||||
|
} |
||||
|
|
||||
|
if currentMinWorkingSet < maxMemoryLimitBytes { |
||||
|
windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) |
||||
|
} |
||||
|
|
||||
|
mBuffer.aligned_ptr = addr_ptr |
||||
|
mBuffer.aligned_length = aligned_length |
||||
|
mBuffer.ptr = addr_ptr + uintptr(diff) |
||||
|
mBuffer.length = length |
||||
|
|
||||
|
slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer)) |
||||
|
slice_header.Data = addr_ptr + uintptr(diff) |
||||
|
slice_header.Len = int(length) |
||||
|
slice_header.Cap = int(length) |
||||
|
|
||||
|
return mBuffer, nil |
||||
|
} |
||||
|
|
||||
|
//typedef struct _MEMORYSTATUSEX {
|
||||
|
// DWORD dwLength;
|
||||
|
// DWORD dwMemoryLoad;
|
||||
|
// DWORDLONG ullTotalPhys;
|
||||
|
// DWORDLONG ullAvailPhys;
|
||||
|
// DWORDLONG ullTotalPageFile;
|
||||
|
// DWORDLONG ullAvailPageFile;
|
||||
|
// DWORDLONG ullTotalVirtual;
|
||||
|
// DWORDLONG ullAvailVirtual;
|
||||
|
// DWORDLONG ullAvailExtendedVirtual;
|
||||
|
// } MEMORYSTATUSEX, *LPMEMORYSTATUSEX;
|
||||
|
//https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
|
||||
|
|
||||
|
type _MEMORYSTATUSEX struct { |
||||
|
dwLength DWORD |
||||
|
dwMemoryLoad DWORD |
||||
|
ullTotalPhys DWORDLONG |
||||
|
ullAvailPhys DWORDLONG |
||||
|
ullTotalPageFile DWORDLONG |
||||
|
ullAvailPageFile DWORDLONG |
||||
|
ullTotalVirtual DWORDLONG |
||||
|
ullAvailVirtual DWORDLONG |
||||
|
ullAvailExtendedVirtual DWORDLONG |
||||
|
} |
||||
|
|
||||
|
// BOOL GlobalMemoryStatusEx(
|
||||
|
// LPMEMORYSTATUSEX lpBuffer
|
||||
|
// );
|
||||
|
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
|
||||
|
func globalMemoryStatusEx() (_MEMORYSTATUSEX, error) { |
||||
|
var mem_status _MEMORYSTATUSEX |
||||
|
|
||||
|
mem_status.dwLength = uint32(unsafe.Sizeof(mem_status)) |
||||
|
_, _, err := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&mem_status))) |
||||
|
|
||||
|
if err != syscall.Errno(0) { |
||||
|
return mem_status, err |
||||
|
} |
||||
|
return mem_status, nil |
||||
|
} |
||||
|
|
||||
|
// typedef struct _SYSTEM_INFO {
|
||||
|
// union {
|
||||
|
// DWORD dwOemId;
|
||||
|
// struct {
|
||||
|
// WORD wProcessorArchitecture;
|
||||
|
// WORD wReserved;
|
||||
|
// };
|
||||
|
// };
|
||||
|
// DWORD dwPageSize;
|
||||
|
// LPVOID lpMinimumApplicationAddress;
|
||||
|
// LPVOID lpMaximumApplicationAddress;
|
||||
|
// DWORD_PTR dwActiveProcessorMask;
|
||||
|
// DWORD dwNumberOfProcessors;
|
||||
|
// DWORD dwProcessorType;
|
||||
|
// DWORD dwAllocationGranularity;
|
||||
|
// WORD wProcessorLevel;
|
||||
|
// WORD wProcessorRevision;
|
||||
|
// } SYSTEM_INFO;
|
||||
|
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info
|
||||
|
type _SYSTEM_INFO struct { |
||||
|
dwOemId DWORD |
||||
|
dwPageSize DWORD |
||||
|
lpMinimumApplicationAddress uintptr |
||||
|
lpMaximumApplicationAddress uintptr |
||||
|
dwActiveProcessorMask uintptr |
||||
|
dwNumberOfProcessors DWORD |
||||
|
dwProcessorType DWORD |
||||
|
dwAllocationGranularity DWORD |
||||
|
wProcessorLevel WORD |
||||
|
wProcessorRevision WORD |
||||
|
} |
||||
|
|
||||
|
// void WINAPI GetSystemInfo(
|
||||
|
// _Out_ LPSYSTEM_INFO lpSystemInfo
|
||||
|
// );
|
||||
|
// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsysteminfo
|
||||
|
func getSystemInfo() (_SYSTEM_INFO, error) { |
||||
|
var si _SYSTEM_INFO |
||||
|
_, _, err := procGetSystemInfo.Call(uintptr(unsafe.Pointer(&si))) |
||||
|
if err != syscall.Errno(0) { |
||||
|
return si, err |
||||
|
} |
||||
|
return si, nil |
||||
|
} |
||||
|
|
||||
|
// BOOL GetProcessWorkingSetSize(
|
||||
|
// HANDLE hProcess,
|
||||
|
// PSIZE_T lpMinimumWorkingSetSize,
|
||||
|
// PSIZE_T lpMaximumWorkingSetSize
|
||||
|
// );
|
||||
|
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getprocessworkingsetsize
|
||||
|
|
||||
|
func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error { |
||||
|
r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet))) |
||||
|
if r1 == 0 { |
||||
|
if err != syscall.Errno(0) { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// BOOL SetProcessWorkingSetSize(
|
||||
|
// HANDLE hProcess,
|
||||
|
// SIZE_T dwMinimumWorkingSetSize,
|
||||
|
// SIZE_T dwMaximumWorkingSetSize
|
||||
|
// );
|
||||
|
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setprocessworkingsetsize
|
||||
|
|
||||
|
func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uint64, dwMaxWorkingSet uint64) error { |
||||
|
r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, uintptr(dwMinWorkingSet), uintptr(dwMaxWorkingSet)) |
||||
|
if r1 == 0 { |
||||
|
if err != syscall.Errno(0) { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
@ -0,0 +1,14 @@ |
|||||
|
package needle |
||||
|
|
||||
|
import "strconv" |
||||
|
|
||||
|
func ReadMemoryMapMaxSizeMB(MemoryMapMaxSizeMBString string) (uint32, error) { |
||||
|
if MemoryMapMaxSizeMBString == "" { |
||||
|
return 0, nil |
||||
|
} |
||||
|
var MemoryMapMaxSize64 uint64 |
||||
|
var err error |
||||
|
MemoryMapMaxSize64, err = strconv.ParseUint(MemoryMapMaxSizeMBString, 10, 32) |
||||
|
MemoryMapMaxSize := uint32(MemoryMapMaxSize64) |
||||
|
return MemoryMapMaxSize, err |
||||
|
} |
||||
@ -0,0 +1,10 @@ |
|||||
|
package needle |
||||
|
|
||||
|
import "testing" |
||||
|
|
||||
|
func TestMemoryMapMaxSizeReadWrite(t *testing.T) { |
||||
|
memoryMapSize, _ := ReadMemoryMapMaxSizeMB("5000") |
||||
|
if memoryMapSize != 5000 { |
||||
|
t.Errorf("empty memoryMapSize:%v", memoryMapSize) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,38 @@ |
|||||
|
// +build windows
|
||||
|
|
||||
|
package storage |
||||
|
|
||||
|
import ( |
||||
|
"os" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/storage/memory_map" |
||||
|
"golang.org/x/sys/windows" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/os_overloads" |
||||
|
) |
||||
|
|
||||
|
func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (*os.File, error) { |
||||
|
|
||||
|
mMap, exists := memory_map.FileMemoryMap[fileName] |
||||
|
if !exists { |
||||
|
|
||||
|
if preallocate > 0 { |
||||
|
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) |
||||
|
} |
||||
|
|
||||
|
if memoryMapSizeMB > 0 { |
||||
|
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true) |
||||
|
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap) |
||||
|
|
||||
|
new_mMap := memory_map.FileMemoryMap[fileName] |
||||
|
new_mMap.CreateMemoryMap(file, 1024*1024*uint64(memoryMapSizeMB)) |
||||
|
return file, e |
||||
|
} else { |
||||
|
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT|windows.O_TRUNC, 0644, false) |
||||
|
return file, e |
||||
|
} |
||||
|
} else { |
||||
|
return mMap.File, nil |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue