Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Refactor to pass memory maps by reference instead of value, fix memory maps not being created properly or written to properly

pull/1087/head
j.laycock 5 years ago
parent
dc50701e7d
commit
840ccdc35d
5 changed files with 39 additions and 39 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 30
      weed/storage/memory_map/memory_map_windows.go
  2. 16
      weed/storage/needle/needle_read_write.go
  3. 8
      weed/storage/volume_create_windows.go
  4. 8
      weed/storage/volume_read_write.go
  5. 16
      weed/storage/volume_super_block.go

30
weed/storage/memory_map/memory_map_windows.go
View File

@ -27,7 +27,7 @@ type MemoryMap struct {
End_Of_File int64
}
var FileMemoryMap = make(map[string]MemoryMap)
var FileMemoryMap = make(map[string]*MemoryMap)
type DWORD = uint32
type WORD = uint16
@ -40,29 +40,27 @@ var system_info, err = getSystemInfo()
var chunk_size = uint64(system_info.dwAllocationGranularity) * 512
func CreateMemoryMap(file *os.File, maxlength uint64) MemoryMap {
func (mem_map *MemoryMap) CreateMemoryMap(file *os.File, maxlength uint64) {
mem_map := MemoryMap{}
maxlength_high := uint32(maxlength >> 32)
maxlength_low := uint32(maxlength & 0xFFFFFFFF)
file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxlength_high, maxlength_low, nil)
if err != nil {
if err == nil {
mem_map.File = file
mem_map.file_memory_map_handle = uintptr(file_memory_map_handle)
mem_map.write_map_views = make([]MemoryBuffer, 0, maxlength/chunk_size)
mem_map.max_length = maxlength
mem_map.End_Of_File = -1
}
return mem_map
}
func DeleteFileAndMemoryMap(mem_map MemoryMap) {
func (mem_map *MemoryMap) DeleteFileAndMemoryMap() {
windows.CloseHandle(windows.Handle(mem_map.file_memory_map_handle))
windows.CloseHandle(windows.Handle(mem_map.File.Fd()))
for _, view := range mem_map.write_map_views {
ReleaseMemory(view)
view.ReleaseMemory()
}
mem_map.write_map_views = nil
@ -76,7 +74,7 @@ func min(x, y uint64) uint64 {
return y
}
func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
func (mem_map *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
for {
if ((offset+length)/chunk_size)+1 > uint64(len(mem_map.write_map_views)) {
@ -92,7 +90,7 @@ func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
data_offset := uint64(0)
for {
write_end := min(remaining_length, chunk_size)
write_end := min((remaining_length + slice_offset), chunk_size)
copy(mem_map.write_map_views[slice_index].Buffer[slice_offset:write_end], data[data_offset:])
remaining_length -= (write_end - slice_offset)
data_offset += (write_end - slice_offset)
@ -105,16 +103,16 @@ func WriteMemory(mem_map MemoryMap, offset uint64, length uint64, data []byte) {
}
}
if mem_map.End_Of_File < int64(offset+length) {
mem_map.End_Of_File = int64(offset + length)
if mem_map.End_Of_File < int64(offset+length-1) {
mem_map.End_Of_File = int64(offset + length - 1)
}
}
func ReadMemory(mem_map MemoryMap, offset uint64, length uint64) (MemoryBuffer, error) {
func (mem_map *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
return allocate(windows.Handle(mem_map.file_memory_map_handle), offset, length, false)
}
func ReleaseMemory(mem_buffer MemoryBuffer) {
func (mem_buffer *MemoryBuffer) ReleaseMemory() {
windows.UnmapViewOfFile(mem_buffer.aligned_ptr)
mem_buffer.ptr = 0
@ -124,9 +122,9 @@ func ReleaseMemory(mem_buffer MemoryBuffer) {
mem_buffer.Buffer = nil
}
func allocateChunk(mem_map MemoryMap) {
func allocateChunk(mem_map *MemoryMap) {
start := uint64(len(mem_map.write_map_views)-1) * chunk_size
start := uint64(len(mem_map.write_map_views)) * chunk_size
mem_buffer, err := allocate(windows.Handle(mem_map.file_memory_map_handle), start, chunk_size, true)
if err == nil {

16
weed/storage/needle/needle_read_write.go
View File

@ -8,10 +8,10 @@ import (
"math"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
. "github.com/joeslay/seaweedfs/weed/storage/types"
"github.com/joeslay/seaweedfs/weed/util"
)
const (
@ -152,7 +152,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32
if err == nil {
if exists {
memory_map.WriteMemory(mem_map, offset, uint64(len(bytesToWrite)), bytesToWrite)
mem_map.WriteMemory(offset, uint64(len(bytesToWrite)), bytesToWrite)
} else {
_, err = w.Write(bytesToWrite)
}
@ -168,9 +168,9 @@ func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dat
mem_map, exists := memory_map.FileMemoryMap[r.Name()]
if exists {
mem_buffer, err := memory_map.ReadMemory(mem_map, uint64(offset), uint64(dataSize))
mem_buffer, err := mem_map.ReadMemory(uint64(offset), uint64(dataSize))
copy(dataSlice, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
return dataSlice, err
} else {
_, err = r.ReadAt(dataSlice, offset)
@ -291,9 +291,9 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, byt
mem_map, exists := memory_map.FileMemoryMap[r.Name()]
if exists {
mem_buffer, err := memory_map.ReadMemory(mem_map, uint64(offset), NeedleHeaderSize)
mem_buffer, err := mem_map.ReadMemory(uint64(offset), NeedleHeaderSize)
copy(bytes, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
if err != nil {
return nil, bytes, 0, err

8
weed/storage/volume_create_windows.go
View File

@ -8,7 +8,7 @@ import (
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"golang.org/x/sys/windows"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/os_overloads"
)
@ -17,8 +17,10 @@ func createVolumeFile(fileName string, preallocate int64) (*os.File, error) {
mem_map, exists := memory_map.FileMemoryMap[fileName]
if !exists {
file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true)
new_mem_map := memory_map.CreateMemoryMap(file, 2^32)
memory_map.FileMemoryMap[fileName] = new_mem_map
memory_map.FileMemoryMap[fileName] = new(memory_map.MemoryMap)
new_mem_map := memory_map.FileMemoryMap[fileName]
new_mem_map.CreateMemoryMap(file, 1024*1024*1024*4)
if preallocate > 0 {
glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)

8
weed/storage/volume_read_write.go
View File

@ -8,10 +8,10 @@ import (
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"github.com/joeslay/seaweedfs/weed/storage/needle"
. "github.com/joeslay/seaweedfs/weed/storage/types"
)
var ErrorNotFound = errors.New("not found")
@ -51,7 +51,7 @@ func (v *Volume) Destroy() (err error) {
}
mem_map, exists := memory_map.FileMemoryMap[v.FileName()]
if exists {
memory_map.DeleteFileAndMemoryMap(mem_map)
mem_map.DeleteFileAndMemoryMap()
delete(memory_map.FileMemoryMap, v.FileName())
}

16
weed/storage/volume_super_block.go
View File

@ -6,11 +6,11 @@ import (
"github.com/joeslay/seaweedfs/weed/storage/memory_map"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"github.com/joeslay/seaweedfs/weed/glog"
"github.com/joeslay/seaweedfs/weed/pb/master_pb"
"github.com/joeslay/seaweedfs/weed/storage/needle"
"github.com/joeslay/seaweedfs/weed/util"
)
const (
@ -73,11 +73,11 @@ func (s *SuperBlock) Bytes() []byte {
func (v *Volume) maybeWriteSuperBlock() error {
mem_map, exists := memory_map.FileMemoryMap[v.FileName()]
mem_map, exists := memory_map.FileMemoryMap[v.dataFile.Name()]
if exists {
if mem_map.End_Of_File == -1 {
v.SuperBlock.version = needle.CurrentVersion
memory_map.WriteMemory(mem_map, 0, uint64(len(v.SuperBlock.Bytes())), v.SuperBlock.Bytes())
mem_map.WriteMemory(0, uint64(len(v.SuperBlock.Bytes())), v.SuperBlock.Bytes())
}
return nil
} else {
@ -113,13 +113,13 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
header := make([]byte, _SuperBlockSize)
mem_map, exists := memory_map.FileMemoryMap[dataFile.Name()]
if exists {
mem_buffer, e := memory_map.ReadMemory(mem_map, 0, _SuperBlockSize)
mem_buffer, e := mem_map.ReadMemory(0, _SuperBlockSize)
if err != nil {
err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e)
return
}
copy(header, mem_buffer.Buffer)
memory_map.ReleaseMemory(mem_buffer)
mem_buffer.ReleaseMemory()
} else {
if _, err = dataFile.Seek(0, 0); err != nil {
err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err)

Write Preview
Loading…
Cancel
Save