Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix isPerfectAppend

pull/664/head
Chris Lu 7 years ago
parent
74332e1a61
commit
c4b92e17d0
2 changed files with 37 additions and 18 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 41
      weed/filesys/dirty_page.go
  2. 14
      weed/filesys/filehandle.go

41
weed/filesys/dirty_page.go
View File

@ -29,21 +29,29 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
pages.Lock() pages.Lock()
defer pages.Unlock() defer pages.Unlock()
isPerfectAppend := len(pages.pages) == 0
isPerfectOverwrite := false
isPerfectAppend := false
if len(pages.pages) > 0 { if len(pages.pages) > 0 {
lastPage := pages.pages[len(pages.pages)-1] lastPage := pages.pages[len(pages.pages)-1]
if lastPage.Offset+int64(len(lastPage.Data)) == offset { if lastPage.Offset+int64(len(lastPage.Data)) == offset {
// write continuous pages // write continuous pages
glog.V(3).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
isPerfectAppend = true isPerfectAppend = true
} }
if pages.pages[0].Offset == offset && pages.totalSize() == int64(len(data)) {
glog.V(4).Infof("%s/%s overwrite [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
isPerfectOverwrite = true
}
} else {
glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
isPerfectAppend = true
} }
isPerfectReplace := false isPerfectReplace := false
for _, page := range pages.pages { for _, page := range pages.pages {
if page.Offset == offset && len(page.Data) == len(data) { if page.Offset == offset && len(page.Data) == len(data) {
// perfect replace // perfect replace
glog.V(3).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
glog.V(4).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
page.Data = data page.Data = data
isPerfectReplace = true isPerfectReplace = true
} }
@ -53,23 +61,34 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
return nil, nil return nil, nil
} }
if isPerfectAppend {
pages.pages = append(pages.pages, &DirtyPage{
Offset: offset,
Data: data,
})
if isPerfectAppend || isPerfectOverwrite {
if isPerfectAppend {
glog.V(4).Infof("%s/%s append2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
pages.pages = append(pages.pages, &DirtyPage{
Offset: offset,
Data: data,
})
}
if isPerfectOverwrite {
glog.V(4).Infof("%s/%s overwrite2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data)))
pages.pages = []*DirtyPage{&DirtyPage{
Offset: offset,
Data: data,
}}
}
if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit {
chunk, err = pages.saveToStorage(ctx) chunk, err = pages.saveToStorage(ctx)
pages.pages = nil pages.pages = nil
glog.V(3).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
glog.V(4).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
} }
return return
} }
chunk, err = pages.saveToStorage(ctx) chunk, err = pages.saveToStorage(ctx)
glog.V(3).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
glog.V(4).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
pages.pages = []*DirtyPage{&DirtyPage{ pages.pages = []*DirtyPage{&DirtyPage{
Offset: offset, Offset: offset,
@ -87,7 +106,7 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f
if chunk, err = pages.saveToStorage(ctx); err == nil { if chunk, err = pages.saveToStorage(ctx); err == nil {
pages.pages = nil pages.pages = nil
if chunk != nil { if chunk != nil {
glog.V(3).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
} }
} }
return return

14
weed/filesys/filehandle.go
View File

@ -39,7 +39,7 @@ var _ = fs.HandleReleaser(&FileHandle{})
func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size))
glog.V(4).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size))
if len(fh.f.Chunks) == 0 { if len(fh.f.Chunks) == 0 {
glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name) glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
@ -73,7 +73,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
}) })
if err != nil { if err != nil {
glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err)
glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err)
return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err) return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
} }
@ -84,7 +84,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
go func(chunkView *filer2.ChunkView) { go func(chunkView *filer2.ChunkView) {
defer wg.Done() defer wg.Done()
glog.V(3).Infof("read fh reading chunk: %+v", chunkView)
glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
locations := vid2Locations[volumeId(chunkView.FileId)] locations := vid2Locations[volumeId(chunkView.FileId)]
if locations == nil || len(locations.Locations) == 0 { if locations == nil || len(locations.Locations) == 0 {
@ -109,7 +109,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
return return
} }
glog.V(3).Infof("read fh read %d bytes: %+v", n, chunkView)
glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
totalRead += n totalRead += n
}(chunkView) }(chunkView)
@ -126,7 +126,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
// write the request to volume servers // write the request to volume servers
glog.V(3).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)))
glog.V(4).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)))
chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data)
if err != nil { if err != nil {
@ -146,7 +146,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
glog.V(3).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name)
glog.V(4).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name)
fh.f.isOpen = false fh.f.isOpen = false
@ -158,7 +158,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
// fflush works at fh level // fflush works at fh level
// send the data to the OS // send the data to the OS
glog.V(3).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req)
glog.V(4).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req)
chunk, err := fh.dirtyPages.FlushToStorage(ctx) chunk, err := fh.dirtyPages.FlushToStorage(ctx)
if err != nil { if err != nil {

Write Preview
Loading…
Cancel
Save