Browse Source
Merge pull request #1087 from joeslay/master
Merge pull request #1087 from joeslay/master
Storing files in in-memory collections on windowspull/1091/head
Chris Lu
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1031 additions and 230 deletions
-
4go.mod
-
3go.sum
-
4weed/command/backup.go
-
2weed/command/compact.go
-
168weed/os_overloads/file_windows.go
-
80weed/os_overloads/syscall_windows.go
-
9weed/os_overloads/types_windows.go
-
1weed/pb/master.proto
-
33weed/pb/master_pb/master.pb.go
-
1weed/pb/volume_server.proto
-
30weed/pb/volume_server_pb/volume_server.pb.go
-
15weed/server/master_grpc_server_volume.go
-
20weed/server/master_server_handlers_admin.go
-
1weed/server/volume_grpc_admin.go
-
2weed/storage/disk_location.go
-
42weed/storage/memory_map/memory_map.go
-
343weed/storage/memory_map/memory_map_windows.go
-
160weed/storage/needle/needle_read_write.go
-
14weed/storage/needle/volume_memory_map_max_size.go
-
10weed/storage/needle/volume_memory_map_max_size_test.go
-
8weed/storage/store.go
-
19weed/storage/volume.go
-
6weed/storage/volume_create.go
-
4weed/storage/volume_create_linux.go
-
38weed/storage/volume_create_windows.go
-
2weed/storage/volume_loading.go
-
7weed/storage/volume_read_write.go
-
67weed/storage/volume_super_block.go
-
138weed/storage/volume_vacuum.go
-
4weed/storage/volume_vacuum_test.go
-
11weed/topology/allocate_volume.go
-
15weed/topology/volume_growth.go
@ -0,0 +1,168 @@ |
|||
// Copyright 2009 The Go Authors. All rights reserved.
|
|||
// Use of this source code is governed by a BSD-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package os_overloads |
|||
|
|||
import ( |
|||
"os" |
|||
"syscall" |
|||
|
|||
"golang.org/x/sys/windows" |
|||
) |
|||
|
|||
func isAbs(path string) (b bool) { |
|||
v := volumeName(path) |
|||
if v == "" { |
|||
return false |
|||
} |
|||
path = path[len(v):] |
|||
if path == "" { |
|||
return false |
|||
} |
|||
return os.IsPathSeparator(path[0]) |
|||
} |
|||
|
|||
func volumeName(path string) (v string) { |
|||
if len(path) < 2 { |
|||
return "" |
|||
} |
|||
// with drive letter
|
|||
c := path[0] |
|||
if path[1] == ':' && |
|||
('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || |
|||
'A' <= c && c <= 'Z') { |
|||
return path[:2] |
|||
} |
|||
// is it UNC
|
|||
if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) && |
|||
!os.IsPathSeparator(path[2]) && path[2] != '.' { |
|||
// first, leading `\\` and next shouldn't be `\`. its server name.
|
|||
for n := 3; n < l-1; n++ { |
|||
// second, next '\' shouldn't be repeated.
|
|||
if os.IsPathSeparator(path[n]) { |
|||
n++ |
|||
// third, following something characters. its share name.
|
|||
if !os.IsPathSeparator(path[n]) { |
|||
if path[n] == '.' { |
|||
break |
|||
} |
|||
for ; n < l; n++ { |
|||
if os.IsPathSeparator(path[n]) { |
|||
break |
|||
} |
|||
} |
|||
return path[:n] |
|||
} |
|||
break |
|||
} |
|||
} |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
// fixLongPath returns the extended-length (\\?\-prefixed) form of
|
|||
// path when needed, in order to avoid the default 260 character file
|
|||
// path limit imposed by Windows. If path is not easily converted to
|
|||
// the extended-length form (for example, if path is a relative path
|
|||
// or contains .. elements), or is short enough, fixLongPath returns
|
|||
// path unmodified.
|
|||
//
|
|||
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
|
|||
func fixLongPath(path string) string { |
|||
// Do nothing (and don't allocate) if the path is "short".
|
|||
// Empirically (at least on the Windows Server 2013 builder),
|
|||
// the kernel is arbitrarily okay with < 248 bytes. That
|
|||
// matches what the docs above say:
|
|||
// "When using an API to create a directory, the specified
|
|||
// path cannot be so long that you cannot append an 8.3 file
|
|||
// name (that is, the directory name cannot exceed MAX_PATH
|
|||
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
|
|||
//
|
|||
// The MSDN docs appear to say that a normal path that is 248 bytes long
|
|||
// will work; empirically the path must be less then 248 bytes long.
|
|||
if len(path) < 248 { |
|||
// Don't fix. (This is how Go 1.7 and earlier worked,
|
|||
// not automatically generating the \\?\ form)
|
|||
return path |
|||
} |
|||
|
|||
// The extended form begins with \\?\, as in
|
|||
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
|
|||
// The extended form disables evaluation of . and .. path
|
|||
// elements and disables the interpretation of / as equivalent
|
|||
// to \. The conversion here rewrites / to \ and elides
|
|||
// . elements as well as trailing or duplicate separators. For
|
|||
// simplicity it avoids the conversion entirely for relative
|
|||
// paths or paths containing .. elements. For now,
|
|||
// \\server\share paths are not converted to
|
|||
// \\?\UNC\server\share paths because the rules for doing so
|
|||
// are less well-specified.
|
|||
if len(path) >= 2 && path[:2] == `\\` { |
|||
// Don't canonicalize UNC paths.
|
|||
return path |
|||
} |
|||
if !isAbs(path) { |
|||
// Relative path
|
|||
return path |
|||
} |
|||
|
|||
const prefix = `\\?` |
|||
|
|||
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) |
|||
copy(pathbuf, prefix) |
|||
n := len(path) |
|||
r, w := 0, len(prefix) |
|||
for r < n { |
|||
switch { |
|||
case os.IsPathSeparator(path[r]): |
|||
// empty block
|
|||
r++ |
|||
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): |
|||
// /./
|
|||
r++ |
|||
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): |
|||
// /../ is currently unhandled
|
|||
return path |
|||
default: |
|||
pathbuf[w] = '\\' |
|||
w++ |
|||
for ; r < n && !os.IsPathSeparator(path[r]); r++ { |
|||
pathbuf[w] = path[r] |
|||
w++ |
|||
} |
|||
} |
|||
} |
|||
// A drive's root directory needs a trailing \
|
|||
if w == len(`\\?\c:`) { |
|||
pathbuf[w] = '\\' |
|||
w++ |
|||
} |
|||
return string(pathbuf[:w]) |
|||
} |
|||
|
|||
// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
|
|||
func syscallMode(i os.FileMode) (o uint32) { |
|||
o |= uint32(i.Perm()) |
|||
if i&os.ModeSetuid != 0 { |
|||
o |= syscall.S_ISUID |
|||
} |
|||
if i&os.ModeSetgid != 0 { |
|||
o |= syscall.S_ISGID |
|||
} |
|||
if i&os.ModeSticky != 0 { |
|||
o |= syscall.S_ISVTX |
|||
} |
|||
// No mapping for Go's ModeTemporary (plan9 only).
|
|||
return |
|||
} |
|||
|
|||
//If the bool is set to true then the file is opened with the parameters FILE_ATTRIBUTE_TEMPORARY and
|
|||
// FILE_FLAG_DELETE_ON_CLOSE
|
|||
func OpenFile(name string, flag int, perm os.FileMode, setToTempAndDelete bool) (file *os.File, err error) { |
|||
r, e := Open(fixLongPath(name), flag|windows.O_CLOEXEC, syscallMode(perm), setToTempAndDelete) |
|||
if e != nil { |
|||
return nil, e |
|||
} |
|||
return os.NewFile(uintptr(r), name), nil |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
// Copyright 2009 The Go Authors. All rights reserved.
|
|||
// Use of this source code is governed by a BSD-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
// Windows system calls.
|
|||
|
|||
package os_overloads |
|||
|
|||
import ( |
|||
"syscall" |
|||
"unsafe" |
|||
|
|||
"golang.org/x/sys/windows" |
|||
) |
|||
|
|||
// windows api calls
|
|||
|
|||
//sys CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) [failretval==InvalidHandle] = CreateFileW
|
|||
|
|||
func makeInheritSa() *syscall.SecurityAttributes { |
|||
var sa syscall.SecurityAttributes |
|||
sa.Length = uint32(unsafe.Sizeof(sa)) |
|||
sa.InheritHandle = 1 |
|||
return &sa |
|||
} |
|||
|
|||
// opens the
|
|||
func Open(path string, mode int, perm uint32, setToTempAndDelete bool) (fd syscall.Handle, err error) { |
|||
if len(path) == 0 { |
|||
return syscall.InvalidHandle, windows.ERROR_FILE_NOT_FOUND |
|||
} |
|||
pathp, err := syscall.UTF16PtrFromString(path) |
|||
if err != nil { |
|||
return syscall.InvalidHandle, err |
|||
} |
|||
var access uint32 |
|||
switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) { |
|||
case windows.O_RDONLY: |
|||
access = windows.GENERIC_READ |
|||
case windows.O_WRONLY: |
|||
access = windows.GENERIC_WRITE |
|||
case windows.O_RDWR: |
|||
access = windows.GENERIC_READ | windows.GENERIC_WRITE |
|||
} |
|||
if mode&windows.O_CREAT != 0 { |
|||
access |= windows.GENERIC_WRITE |
|||
} |
|||
if mode&windows.O_APPEND != 0 { |
|||
access &^= windows.GENERIC_WRITE |
|||
access |= windows.FILE_APPEND_DATA |
|||
} |
|||
sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE) |
|||
var sa *syscall.SecurityAttributes |
|||
if mode&windows.O_CLOEXEC == 0 { |
|||
sa = makeInheritSa() |
|||
} |
|||
var createmode uint32 |
|||
switch { |
|||
case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL): |
|||
createmode = windows.CREATE_NEW |
|||
case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC): |
|||
createmode = windows.CREATE_ALWAYS |
|||
case mode&windows.O_CREAT == windows.O_CREAT: |
|||
createmode = windows.OPEN_ALWAYS |
|||
case mode&windows.O_TRUNC == windows.O_TRUNC: |
|||
createmode = windows.TRUNCATE_EXISTING |
|||
default: |
|||
createmode = windows.OPEN_EXISTING |
|||
} |
|||
|
|||
var h syscall.Handle |
|||
var e error |
|||
|
|||
if setToTempAndDelete { |
|||
h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, (windows.FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE), 0) |
|||
} else { |
|||
h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, windows.FILE_ATTRIBUTE_NORMAL, 0) |
|||
} |
|||
return h, e |
|||
} |
|||
@ -0,0 +1,9 @@ |
|||
// Copyright 2011 The Go Authors. All rights reserved.
|
|||
// Use of this source code is governed by a BSD-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package os_overloads |
|||
|
|||
const ( |
|||
FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 |
|||
) |
|||
@ -0,0 +1,42 @@ |
|||
// +build !windows
|
|||
|
|||
package memory_map |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
) |
|||
|
|||
type MemoryBuffer struct { |
|||
aligned_length uint64 |
|||
length uint64 |
|||
aligned_ptr uintptr |
|||
ptr uintptr |
|||
Buffer []byte |
|||
} |
|||
|
|||
type MemoryMap struct { |
|||
File *os.File |
|||
file_memory_map_handle uintptr |
|||
write_map_views []MemoryBuffer |
|||
max_length uint64 |
|||
End_of_file int64 |
|||
} |
|||
|
|||
var FileMemoryMap = make(map[string]*MemoryMap) |
|||
|
|||
func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { |
|||
} |
|||
|
|||
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { |
|||
|
|||
} |
|||
|
|||
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) ([]byte, error) { |
|||
dataSlice := []byte{} |
|||
return dataSlice, fmt.Errorf("Memory Map not implemented for this platform") |
|||
} |
|||
|
|||
func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() { |
|||
|
|||
} |
|||
@ -0,0 +1,343 @@ |
|||
// +build windows
|
|||
|
|||
package memory_map |
|||
|
|||
import ( |
|||
"os" |
|||
"reflect" |
|||
"syscall" |
|||
"unsafe" |
|||
|
|||
"golang.org/x/sys/windows" |
|||
) |
|||
|
|||
type MemoryBuffer struct { |
|||
aligned_length uint64 |
|||
length uint64 |
|||
aligned_ptr uintptr |
|||
ptr uintptr |
|||
Buffer []byte |
|||
} |
|||
|
|||
type MemoryMap struct { |
|||
File *os.File |
|||
file_memory_map_handle uintptr |
|||
write_map_views []MemoryBuffer |
|||
max_length uint64 |
|||
End_of_file int64 |
|||
} |
|||
|
|||
var FileMemoryMap = make(map[string]*MemoryMap) |
|||
|
|||
type DWORDLONG = uint64 |
|||
type DWORD = uint32 |
|||
type WORD = uint16 |
|||
|
|||
var ( |
|||
modkernel32 = syscall.NewLazyDLL("kernel32.dll") |
|||
|
|||
procGetSystemInfo = modkernel32.NewProc("GetSystemInfo") |
|||
procGlobalMemoryStatusEx = modkernel32.NewProc("GlobalMemoryStatusEx") |
|||
procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize") |
|||
procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize") |
|||
) |
|||
|
|||
var currentProcess, _ = windows.GetCurrentProcess() |
|||
var currentMinWorkingSet uint64 = 0 |
|||
var currentMaxWorkingSet uint64 = 0 |
|||
var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet) |
|||
|
|||
var systemInfo, _ = getSystemInfo() |
|||
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 128 |
|||
|
|||
var memoryStatusEx, _ = globalMemoryStatusEx() |
|||
var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8) |
|||
|
|||
func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { |
|||
|
|||
chunks := (maxLength / chunkSize) |
|||
if chunks*chunkSize < maxLength { |
|||
chunks = chunks + 1 |
|||
} |
|||
|
|||
alignedMaxLength := chunks * chunkSize |
|||
|
|||
maxLength_high := uint32(alignedMaxLength >> 32) |
|||
maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF) |
|||
file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil) |
|||
|
|||
if err == nil { |
|||
mMap.File = file |
|||
mMap.file_memory_map_handle = uintptr(file_memory_map_handle) |
|||
mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize) |
|||
mMap.max_length = alignedMaxLength |
|||
mMap.End_of_file = -1 |
|||
} |
|||
} |
|||
|
|||
func (mMap *MemoryMap) DeleteFileAndMemoryMap() { |
|||
//First we close the file handles first to delete the file,
|
|||
//Then we unmap the memory to ensure the unmapping process doesn't write the data to disk
|
|||
windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle)) |
|||
windows.CloseHandle(windows.Handle(mMap.File.Fd())) |
|||
|
|||
for _, view := range mMap.write_map_views { |
|||
view.releaseMemory() |
|||
} |
|||
|
|||
mMap.write_map_views = nil |
|||
mMap.max_length = 0 |
|||
} |
|||
|
|||
func min(x, y uint64) uint64 { |
|||
if x < y { |
|||
return x |
|||
} |
|||
return y |
|||
} |
|||
|
|||
func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { |
|||
|
|||
for { |
|||
if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) { |
|||
allocateChunk(mMap) |
|||
} else { |
|||
break |
|||
} |
|||
} |
|||
|
|||
remaining_length := length |
|||
sliceIndex := offset / chunkSize |
|||
sliceOffset := offset - (sliceIndex * chunkSize) |
|||
dataOffset := uint64(0) |
|||
|
|||
for { |
|||
writeEnd := min((remaining_length + sliceOffset), chunkSize) |
|||
copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:]) |
|||
remaining_length -= (writeEnd - sliceOffset) |
|||
dataOffset += (writeEnd - sliceOffset) |
|||
|
|||
if remaining_length > 0 { |
|||
sliceIndex += 1 |
|||
sliceOffset = 0 |
|||
} else { |
|||
break |
|||
} |
|||
} |
|||
|
|||
if mMap.End_of_file < int64(offset+length-1) { |
|||
mMap.End_of_file = int64(offset + length - 1) |
|||
} |
|||
} |
|||
|
|||
func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (dataSlice []byte, err error) { |
|||
dataSlice = make([]byte, length) |
|||
mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false) |
|||
copy(dataSlice, mBuffer.Buffer) |
|||
mBuffer.releaseMemory() |
|||
return dataSlice, err |
|||
} |
|||
|
|||
func (mBuffer *MemoryBuffer) releaseMemory() { |
|||
|
|||
windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) |
|||
windows.UnmapViewOfFile(mBuffer.aligned_ptr) |
|||
|
|||
currentMinWorkingSet -= mBuffer.aligned_length |
|||
currentMaxWorkingSet -= mBuffer.aligned_length |
|||
|
|||
if currentMinWorkingSet < maxMemoryLimitBytes { |
|||
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) |
|||
} |
|||
|
|||
mBuffer.ptr = 0 |
|||
mBuffer.aligned_ptr = 0 |
|||
mBuffer.length = 0 |
|||
mBuffer.aligned_length = 0 |
|||
mBuffer.Buffer = nil |
|||
} |
|||
|
|||
func allocateChunk(mMap *MemoryMap) { |
|||
start := uint64(len(mMap.write_map_views)) * chunkSize |
|||
mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true) |
|||
|
|||
if err == nil { |
|||
mMap.write_map_views = append(mMap.write_map_views, mBuffer) |
|||
} |
|||
} |
|||
|
|||
func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) { |
|||
|
|||
mBuffer := MemoryBuffer{} |
|||
|
|||
//align memory allocations to the minium virtal memory allocation size
|
|||
dwSysGran := systemInfo.dwAllocationGranularity |
|||
|
|||
start := (offset / uint64(dwSysGran)) * uint64(dwSysGran) |
|||
diff := offset - start |
|||
aligned_length := diff + length |
|||
|
|||
offset_high := uint32(start >> 32) |
|||
offset_low := uint32(start & 0xFFFFFFFF) |
|||
|
|||
access := windows.FILE_MAP_READ |
|||
|
|||
if write { |
|||
access = windows.FILE_MAP_WRITE |
|||
} |
|||
|
|||
currentMinWorkingSet += aligned_length |
|||
currentMaxWorkingSet += aligned_length |
|||
|
|||
if currentMinWorkingSet < maxMemoryLimitBytes { |
|||
// increase the process working set size to hint to windows memory manager to
|
|||
// prioritise keeping this memory mapped in physical memory over other standby memory
|
|||
var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) |
|||
} |
|||
|
|||
addr_ptr, errno := windows.MapViewOfFile(hMapFile, |
|||
uint32(access), // read/write permission
|
|||
offset_high, |
|||
offset_low, |
|||
uintptr(aligned_length)) |
|||
|
|||
if addr_ptr == 0 { |
|||
return mBuffer, errno |
|||
} |
|||
|
|||
if currentMinWorkingSet < maxMemoryLimitBytes { |
|||
windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) |
|||
} |
|||
|
|||
mBuffer.aligned_ptr = addr_ptr |
|||
mBuffer.aligned_length = aligned_length |
|||
mBuffer.ptr = addr_ptr + uintptr(diff) |
|||
mBuffer.length = length |
|||
|
|||
slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer)) |
|||
slice_header.Data = addr_ptr + uintptr(diff) |
|||
slice_header.Len = int(length) |
|||
slice_header.Cap = int(length) |
|||
|
|||
return mBuffer, nil |
|||
} |
|||
|
|||
//typedef struct _MEMORYSTATUSEX {
|
|||
// DWORD dwLength;
|
|||
// DWORD dwMemoryLoad;
|
|||
// DWORDLONG ullTotalPhys;
|
|||
// DWORDLONG ullAvailPhys;
|
|||
// DWORDLONG ullTotalPageFile;
|
|||
// DWORDLONG ullAvailPageFile;
|
|||
// DWORDLONG ullTotalVirtual;
|
|||
// DWORDLONG ullAvailVirtual;
|
|||
// DWORDLONG ullAvailExtendedVirtual;
|
|||
// } MEMORYSTATUSEX, *LPMEMORYSTATUSEX;
|
|||
//https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
|
|||
|
|||
type _MEMORYSTATUSEX struct { |
|||
dwLength DWORD |
|||
dwMemoryLoad DWORD |
|||
ullTotalPhys DWORDLONG |
|||
ullAvailPhys DWORDLONG |
|||
ullTotalPageFile DWORDLONG |
|||
ullAvailPageFile DWORDLONG |
|||
ullTotalVirtual DWORDLONG |
|||
ullAvailVirtual DWORDLONG |
|||
ullAvailExtendedVirtual DWORDLONG |
|||
} |
|||
|
|||
// BOOL GlobalMemoryStatusEx(
|
|||
// LPMEMORYSTATUSEX lpBuffer
|
|||
// );
|
|||
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
|
|||
func globalMemoryStatusEx() (_MEMORYSTATUSEX, error) { |
|||
var mem_status _MEMORYSTATUSEX |
|||
|
|||
mem_status.dwLength = uint32(unsafe.Sizeof(mem_status)) |
|||
_, _, err := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&mem_status))) |
|||
|
|||
if err != syscall.Errno(0) { |
|||
return mem_status, err |
|||
} |
|||
return mem_status, nil |
|||
} |
|||
|
|||
// typedef struct _SYSTEM_INFO {
|
|||
// union {
|
|||
// DWORD dwOemId;
|
|||
// struct {
|
|||
// WORD wProcessorArchitecture;
|
|||
// WORD wReserved;
|
|||
// };
|
|||
// };
|
|||
// DWORD dwPageSize;
|
|||
// LPVOID lpMinimumApplicationAddress;
|
|||
// LPVOID lpMaximumApplicationAddress;
|
|||
// DWORD_PTR dwActiveProcessorMask;
|
|||
// DWORD dwNumberOfProcessors;
|
|||
// DWORD dwProcessorType;
|
|||
// DWORD dwAllocationGranularity;
|
|||
// WORD wProcessorLevel;
|
|||
// WORD wProcessorRevision;
|
|||
// } SYSTEM_INFO;
|
|||
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info
|
|||
type _SYSTEM_INFO struct { |
|||
dwOemId DWORD |
|||
dwPageSize DWORD |
|||
lpMinimumApplicationAddress uintptr |
|||
lpMaximumApplicationAddress uintptr |
|||
dwActiveProcessorMask uintptr |
|||
dwNumberOfProcessors DWORD |
|||
dwProcessorType DWORD |
|||
dwAllocationGranularity DWORD |
|||
wProcessorLevel WORD |
|||
wProcessorRevision WORD |
|||
} |
|||
|
|||
// void WINAPI GetSystemInfo(
|
|||
// _Out_ LPSYSTEM_INFO lpSystemInfo
|
|||
// );
|
|||
// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsysteminfo
|
|||
func getSystemInfo() (_SYSTEM_INFO, error) { |
|||
var si _SYSTEM_INFO |
|||
_, _, err := procGetSystemInfo.Call(uintptr(unsafe.Pointer(&si))) |
|||
if err != syscall.Errno(0) { |
|||
return si, err |
|||
} |
|||
return si, nil |
|||
} |
|||
|
|||
// BOOL GetProcessWorkingSetSize(
|
|||
// HANDLE hProcess,
|
|||
// PSIZE_T lpMinimumWorkingSetSize,
|
|||
// PSIZE_T lpMaximumWorkingSetSize
|
|||
// );
|
|||
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getprocessworkingsetsize
|
|||
|
|||
func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error { |
|||
r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet))) |
|||
if r1 == 0 { |
|||
if err != syscall.Errno(0) { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// BOOL SetProcessWorkingSetSize(
|
|||
// HANDLE hProcess,
|
|||
// SIZE_T dwMinimumWorkingSetSize,
|
|||
// SIZE_T dwMaximumWorkingSetSize
|
|||
// );
|
|||
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setprocessworkingsetsize
|
|||
|
|||
func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uint64, dwMaxWorkingSet uint64) error { |
|||
r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, uintptr(dwMinWorkingSet), uintptr(dwMaxWorkingSet)) |
|||
if r1 == 0 { |
|||
if err != syscall.Errno(0) { |
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
package needle |
|||
|
|||
import "strconv" |
|||
|
|||
func ReadMemoryMapMaxSizeMB(MemoryMapMaxSizeMBString string) (uint32, error) { |
|||
if MemoryMapMaxSizeMBString == "" { |
|||
return 0, nil |
|||
} |
|||
var MemoryMapMaxSize64 uint64 |
|||
var err error |
|||
MemoryMapMaxSize64, err = strconv.ParseUint(MemoryMapMaxSizeMBString, 10, 32) |
|||
MemoryMapMaxSize := uint32(MemoryMapMaxSize64) |
|||
return MemoryMapMaxSize, err |
|||
} |
|||
@ -0,0 +1,10 @@ |
|||
package needle |
|||
|
|||
import "testing" |
|||
|
|||
func TestMemoryMapMaxSizeReadWrite(t *testing.T) { |
|||
memoryMapSize, _ := ReadMemoryMapMaxSizeMB("5000") |
|||
if memoryMapSize != 5000 { |
|||
t.Errorf("empty memoryMapSize:%v", memoryMapSize) |
|||
} |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
// +build windows
|
|||
|
|||
package storage |
|||
|
|||
import ( |
|||
"os" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/storage/memory_map" |
|||
"golang.org/x/sys/windows" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/os_overloads" |
|||
) |
|||
|
|||
func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (*os.File, error) { |
|||
|
|||
mMap, exists := memory_map.FileMemoryMap[fileName] |
|||
if !exists { |
|||
|
|||
if preallocate > 0 { |
|||
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) |
|||
} |
|||
|
|||
if memoryMapSizeMB > 0 { |
|||
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true) |
|||
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap) |
|||
|
|||
new_mMap := memory_map.FileMemoryMap[fileName] |
|||
new_mMap.CreateMemoryMap(file, 1024*1024*uint64(memoryMapSizeMB)) |
|||
return file, e |
|||
} else { |
|||
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT|windows.O_TRUNC, 0644, false) |
|||
return file, e |
|||
} |
|||
} else { |
|||
return mMap.File, nil |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue