hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 870 additions and 100 deletions
-
114.github/workflows/container_latest.yml
-
118.github/workflows/container_release.yml
-
2README.md
-
6docker/Dockerfile.go_build
-
6docker/Dockerfile.go_build_large
-
2docker/compose/local-mount-profile-compose.yml
-
6go.mod
-
2go.sum
-
4k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
60weed/command/filer_copy.go
-
2weed/filer/filerstore_wrapper.go
-
6weed/filer/reader_at.go
-
2weed/filesys/dir.go
-
9weed/filesys/dir_rename.go
-
3weed/filesys/dirty_page_interval.go
-
10weed/filesys/dirty_pages.go
-
32weed/filesys/dirty_pages_continuous.go
-
166weed/filesys/dirty_pages_temp_file.go
-
289weed/filesys/dirty_pages_temp_interval.go
-
10weed/filesys/file.go
-
25weed/filesys/filehandle.go
-
2weed/filesys/meta_cache/meta_cache.go
-
7weed/filesys/wfs.go
-
2weed/s3api/s3err/s3api_errors.go
-
16weed/server/common.go
-
2weed/server/volume_grpc_copy.go
-
13weed/statik/statik.go
-
4weed/storage/volume_vacuum.go
-
4weed/topology/node.go
-
8weed/topology/topology.go
-
11weed/topology/topology_event_handling.go
-
6weed/topology/volume_layout.go
-
2weed/util/bounded_tree/bounded_tree.go
-
2weed/util/constants.go
-
13weed/weed.go
@ -0,0 +1,114 @@ |
|||
name: Build Latest Containers |
|||
on: |
|||
push: |
|||
branches: |
|||
- master |
|||
workflow_dispatch: [] |
|||
|
|||
jobs: |
|||
build-latest: |
|||
runs-on: [ubuntu-latest] |
|||
steps: |
|||
- |
|||
name: Checkout |
|||
uses: actions/checkout@v2 |
|||
- |
|||
name: Docker meta |
|||
id: docker_meta |
|||
uses: crazy-max/ghaction-docker-meta@v2 |
|||
with: |
|||
images: | |
|||
chrislusf/seaweedfs |
|||
ghcr.io/chrislusf/seaweedfs |
|||
tags: | |
|||
type=raw,value=latest |
|||
labels: | |
|||
org.opencontainers.image.title=seaweedfs |
|||
org.opencontainers.image.vendor=Chris Lu |
|||
- |
|||
name: Set up QEMU |
|||
uses: docker/setup-qemu-action@v1 |
|||
- |
|||
name: Set up Docker Buildx |
|||
uses: docker/setup-buildx-action@v1 |
|||
with: |
|||
buildkitd-flags: "--debug" |
|||
- |
|||
name: Login to Docker Hub |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
username: ${{ secrets.DOCKER_USERNAME }} |
|||
password: ${{ secrets.DOCKER_PASSWORD }} |
|||
- |
|||
name: Login to GHCR |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
registry: ghcr.io |
|||
username: ${{ secrets.GHCR_USERNAME }} |
|||
password: ${{ secrets.GHCR_TOKEN }} |
|||
- |
|||
name: Build |
|||
uses: docker/build-push-action@v2 |
|||
with: |
|||
context: ./docker |
|||
push: ${{ github.event_name != 'pull_request' }} |
|||
file: ./docker/Dockerfile |
|||
platforms: linux/amd64 |
|||
tags: ${{ steps.docker_meta.outputs.tags }} |
|||
labels: ${{ steps.docker_meta.outputs.labels }} |
|||
|
|||
build-dev: |
|||
runs-on: [ubuntu-latest] |
|||
steps: |
|||
- |
|||
name: Checkout |
|||
uses: actions/checkout@v2 |
|||
- |
|||
name: Docker meta |
|||
id: docker_meta |
|||
uses: crazy-max/ghaction-docker-meta@v2 |
|||
with: |
|||
images: | |
|||
chrislusf/seaweedfs |
|||
ghcr.io/chrislusf/seaweedfs |
|||
tags: | |
|||
type=raw,value=dev |
|||
labels: | |
|||
org.opencontainers.image.title=seaweedfs |
|||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
|||
org.opencontainers.image.vendor=Chris Lu |
|||
- |
|||
name: Set up QEMU |
|||
uses: docker/setup-qemu-action@v1 |
|||
- |
|||
name: Set up Docker Buildx |
|||
uses: docker/setup-buildx-action@v1 |
|||
with: |
|||
buildkitd-flags: "--debug" |
|||
- |
|||
name: Login to Docker Hub |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
username: ${{ secrets.DOCKER_USERNAME }} |
|||
password: ${{ secrets.DOCKER_PASSWORD }} |
|||
- |
|||
name: Login to GHCR |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
registry: ghcr.io |
|||
username: ${{ secrets.GHCR_USERNAME }} |
|||
password: ${{ secrets.GHCR_TOKEN }} |
|||
- |
|||
name: Build |
|||
uses: docker/build-push-action@v2 |
|||
with: |
|||
context: ./docker |
|||
push: ${{ github.event_name != 'pull_request' }} |
|||
file: ./docker/Dockerfile.go_build |
|||
platforms: linux/amd64 |
|||
tags: ${{ steps.docker_meta.outputs.tags }} |
|||
labels: ${{ steps.docker_meta.outputs.labels }} |
|||
@ -0,0 +1,118 @@ |
|||
name: Build Release Containers |
|||
on: |
|||
push: |
|||
tags: |
|||
- '*' |
|||
workflow_dispatch: [] |
|||
|
|||
jobs: |
|||
build-default: |
|||
runs-on: [ubuntu-latest] |
|||
steps: |
|||
- |
|||
name: Checkout |
|||
uses: actions/checkout@v2 |
|||
- |
|||
name: Docker meta |
|||
id: docker_meta |
|||
uses: crazy-max/ghaction-docker-meta@v2 |
|||
with: |
|||
images: | |
|||
chrislusf/seaweedfs |
|||
ghcr.io/chrislusf/seaweedfs |
|||
tags: | |
|||
type=ref,event=tag |
|||
flavor: | |
|||
latest=false |
|||
labels: | |
|||
org.opencontainers.image.title=seaweedfs |
|||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
|||
org.opencontainers.image.vendor=Chris Lu |
|||
- |
|||
name: Set up QEMU |
|||
uses: docker/setup-qemu-action@v1 |
|||
- |
|||
name: Set up Docker Buildx |
|||
uses: docker/setup-buildx-action@v1 |
|||
with: |
|||
buildkitd-flags: "--debug" |
|||
- |
|||
name: Login to Docker Hub |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
username: ${{ secrets.DOCKER_USERNAME }} |
|||
password: ${{ secrets.DOCKER_PASSWORD }} |
|||
- |
|||
name: Login to GHCR |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
registry: ghcr.io |
|||
username: ${{ secrets.GHCR_USERNAME }} |
|||
password: ${{ secrets.GHCR_TOKEN }} |
|||
- |
|||
name: Build |
|||
uses: docker/build-push-action@v2 |
|||
with: |
|||
context: ./docker |
|||
push: ${{ github.event_name != 'pull_request' }} |
|||
file: ./docker/Dockerfile.go_build |
|||
platforms: linux/amd64 |
|||
tags: ${{ steps.docker_meta.outputs.tags }} |
|||
labels: ${{ steps.docker_meta.outputs.labels }} |
|||
build-large: |
|||
runs-on: [ubuntu-latest] |
|||
steps: |
|||
- |
|||
name: Checkout |
|||
uses: actions/checkout@v2 |
|||
- |
|||
name: Docker meta |
|||
id: docker_meta |
|||
uses: crazy-max/ghaction-docker-meta@v2 |
|||
with: |
|||
images: | |
|||
chrislusf/seaweedfs |
|||
ghcr.io/chrislusf/seaweedfs |
|||
tags: | |
|||
type=ref,event=tag,suffix=_large_disk |
|||
flavor: | |
|||
latest=false |
|||
labels: | |
|||
org.opencontainers.image.title=seaweedfs |
|||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
|||
org.opencontainers.image.vendor=Chris Lu |
|||
- |
|||
name: Set up QEMU |
|||
uses: docker/setup-qemu-action@v1 |
|||
- |
|||
name: Set up Docker Buildx |
|||
uses: docker/setup-buildx-action@v1 |
|||
with: |
|||
buildkitd-flags: "--debug" |
|||
- |
|||
name: Login to Docker Hub |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
username: ${{ secrets.DOCKER_USERNAME }} |
|||
password: ${{ secrets.DOCKER_PASSWORD }} |
|||
- |
|||
name: Login to GHCR |
|||
if: github.event_name != 'pull_request' |
|||
uses: docker/login-action@v1 |
|||
with: |
|||
registry: ghcr.io |
|||
username: ${{ secrets.GHCR_USERNAME }} |
|||
password: ${{ secrets.GHCR_TOKEN }} |
|||
- |
|||
name: Build |
|||
uses: docker/build-push-action@v2 |
|||
with: |
|||
context: ./docker |
|||
push: ${{ github.event_name != 'pull_request' }} |
|||
file: ./docker/Dockerfile.go_build_large |
|||
platforms: linux/amd64 |
|||
tags: ${{ steps.docker_meta.outputs.tags }} |
|||
labels: ${{ steps.docker_meta.outputs.labels }} |
|||
@ -1,5 +1,5 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
appVersion: "2.43" |
|||
version: 2.43 |
|||
appVersion: "2.48" |
|||
version: 2.48 |
|||
@ -0,0 +1,10 @@ |
|||
package filesys |
|||
|
|||
type DirtyPages interface { |
|||
AddPage(offset int64, data []byte) |
|||
FlushData() error |
|||
ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) |
|||
GetStorageOptions() (collection, replication string) |
|||
SetWriteOnly(writeOnly bool) |
|||
GetWriteOnly() (writeOnly bool) |
|||
} |
|||
@ -0,0 +1,166 @@ |
|||
package filesys |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"io" |
|||
"os" |
|||
"path/filepath" |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
type TempFileDirtyPages struct { |
|||
f *File |
|||
tf *os.File |
|||
writtenIntervals *WrittenContinuousIntervals |
|||
writeOnly bool |
|||
writeWaitGroup sync.WaitGroup |
|||
pageAddLock sync.Mutex |
|||
chunkAddLock sync.Mutex |
|||
lastErr error |
|||
collection string |
|||
replication string |
|||
} |
|||
|
|||
var ( |
|||
tmpDir = filepath.Join(os.TempDir(), "sw") |
|||
) |
|||
|
|||
func init() { |
|||
os.Mkdir(tmpDir, 0755) |
|||
} |
|||
|
|||
func newTempFileDirtyPages(file *File, writeOnly bool) *TempFileDirtyPages { |
|||
|
|||
tempFile := &TempFileDirtyPages{ |
|||
f: file, |
|||
writeOnly: writeOnly, |
|||
writtenIntervals: &WrittenContinuousIntervals{}, |
|||
} |
|||
|
|||
return tempFile |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) AddPage(offset int64, data []byte) { |
|||
|
|||
pages.pageAddLock.Lock() |
|||
defer pages.pageAddLock.Unlock() |
|||
|
|||
if pages.tf == nil { |
|||
tf, err := os.CreateTemp(tmpDir, "") |
|||
if err != nil { |
|||
glog.Errorf("create temp file: %v", err) |
|||
pages.lastErr = err |
|||
return |
|||
} |
|||
pages.tf = tf |
|||
pages.writtenIntervals.tempFile = tf |
|||
pages.writtenIntervals.lastOffset = 0 |
|||
} |
|||
|
|||
writtenOffset := pages.writtenIntervals.lastOffset |
|||
dataSize := int64(len(data)) |
|||
|
|||
// glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+dataSize)
|
|||
|
|||
if _, err := pages.tf.WriteAt(data, writtenOffset); err != nil { |
|||
pages.lastErr = err |
|||
} else { |
|||
pages.writtenIntervals.AddInterval(writtenOffset, len(data), offset) |
|||
pages.writtenIntervals.lastOffset += dataSize |
|||
} |
|||
|
|||
// pages.writtenIntervals.debug()
|
|||
|
|||
return |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) FlushData() error { |
|||
|
|||
pages.saveExistingPagesToStorage() |
|||
pages.writeWaitGroup.Wait() |
|||
if pages.lastErr != nil { |
|||
return fmt.Errorf("flush data: %v", pages.lastErr) |
|||
} |
|||
pages.pageAddLock.Lock() |
|||
defer pages.pageAddLock.Unlock() |
|||
if pages.tf != nil { |
|||
|
|||
pages.writtenIntervals.tempFile = nil |
|||
pages.writtenIntervals.lists = nil |
|||
|
|||
pages.tf.Close() |
|||
os.Remove(pages.tf.Name()) |
|||
pages.tf = nil |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) saveExistingPagesToStorage() { |
|||
|
|||
pageSize := pages.f.wfs.option.ChunkSizeLimit |
|||
|
|||
// glog.V(4).Infof("%v saveExistingPagesToStorage %d lists", pages.f.Name, len(pages.writtenIntervals.lists))
|
|||
|
|||
for _, list := range pages.writtenIntervals.lists { |
|||
listStopOffset := list.Offset() + list.Size() |
|||
for uploadedOffset:=int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize { |
|||
start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize) |
|||
if start >= stop { |
|||
continue |
|||
} |
|||
// glog.V(4).Infof("uploading %v [%d,%d) %d/%d", pages.f.Name, start, stop, i, len(pages.writtenIntervals.lists))
|
|||
pages.saveToStorage(list.ToReader(start, stop), start, stop-start) |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) { |
|||
|
|||
mtime := time.Now().UnixNano() |
|||
pages.writeWaitGroup.Add(1) |
|||
writer := func() { |
|||
defer pages.writeWaitGroup.Done() |
|||
|
|||
reader = io.LimitReader(reader, size) |
|||
chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset) |
|||
if err != nil { |
|||
glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err) |
|||
pages.lastErr = err |
|||
return |
|||
} |
|||
chunk.Mtime = mtime |
|||
pages.collection, pages.replication = collection, replication |
|||
pages.chunkAddLock.Lock() |
|||
defer pages.chunkAddLock.Unlock() |
|||
pages.f.addChunks([]*filer_pb.FileChunk{chunk}) |
|||
glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.f.fullpath(), chunk.FileId, offset, offset+size) |
|||
} |
|||
|
|||
if pages.f.wfs.concurrentWriters != nil { |
|||
pages.f.wfs.concurrentWriters.Execute(writer) |
|||
} else { |
|||
go writer() |
|||
} |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { |
|||
return pages.writtenIntervals.ReadDataAt(data, startOffset) |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) GetStorageOptions() (collection, replication string) { |
|||
return pages.collection, pages.replication |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) SetWriteOnly(writeOnly bool) { |
|||
if pages.writeOnly { |
|||
pages.writeOnly = writeOnly |
|||
} |
|||
} |
|||
|
|||
func (pages *TempFileDirtyPages) GetWriteOnly() (writeOnly bool) { |
|||
return pages.writeOnly |
|||
} |
|||
@ -0,0 +1,289 @@ |
|||
package filesys |
|||
|
|||
import ( |
|||
"io" |
|||
"log" |
|||
"os" |
|||
) |
|||
|
|||
type WrittenIntervalNode struct { |
|||
DataOffset int64 |
|||
TempOffset int64 |
|||
Size int64 |
|||
Next *WrittenIntervalNode |
|||
} |
|||
|
|||
type WrittenIntervalLinkedList struct { |
|||
tempFile *os.File |
|||
Head *WrittenIntervalNode |
|||
Tail *WrittenIntervalNode |
|||
} |
|||
|
|||
type WrittenContinuousIntervals struct { |
|||
tempFile *os.File |
|||
lastOffset int64 |
|||
lists []*WrittenIntervalLinkedList |
|||
} |
|||
|
|||
func (list *WrittenIntervalLinkedList) Offset() int64 { |
|||
return list.Head.DataOffset |
|||
} |
|||
func (list *WrittenIntervalLinkedList) Size() int64 { |
|||
return list.Tail.DataOffset + list.Tail.Size - list.Head.DataOffset |
|||
} |
|||
func (list *WrittenIntervalLinkedList) addNodeToTail(node *WrittenIntervalNode) { |
|||
// glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
|
|||
if list.Tail.TempOffset+list.Tail.Size == node.TempOffset { |
|||
// already connected
|
|||
list.Tail.Size += node.Size |
|||
} else { |
|||
list.Tail.Next = node |
|||
list.Tail = node |
|||
} |
|||
} |
|||
func (list *WrittenIntervalLinkedList) addNodeToHead(node *WrittenIntervalNode) { |
|||
// glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
|
|||
node.Next = list.Head |
|||
list.Head = node |
|||
} |
|||
|
|||
func (list *WrittenIntervalLinkedList) ReadData(buf []byte, start, stop int64) { |
|||
t := list.Head |
|||
for { |
|||
|
|||
nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) |
|||
if nodeStart < nodeStop { |
|||
// glog.V(4).Infof("copying start=%d stop=%d t=[%d,%d) => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.DataOffset, t.DataOffset+t.Size, len(buf), nodeStart, nodeStop)
|
|||
list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset + nodeStart - t.DataOffset) |
|||
} |
|||
|
|||
if t.Next == nil { |
|||
break |
|||
} |
|||
t = t.Next |
|||
} |
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) TotalSize() (total int64) { |
|||
for _, list := range c.lists { |
|||
total += list.Size() |
|||
} |
|||
return |
|||
} |
|||
|
|||
func (list *WrittenIntervalLinkedList) subList(start, stop int64) *WrittenIntervalLinkedList { |
|||
var nodes []*WrittenIntervalNode |
|||
for t := list.Head; t != nil; t = t.Next { |
|||
nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) |
|||
if nodeStart >= nodeStop { |
|||
// skip non overlapping WrittenIntervalNode
|
|||
continue |
|||
} |
|||
nodes = append(nodes, &WrittenIntervalNode{ |
|||
TempOffset: t.TempOffset + nodeStart - t.DataOffset, |
|||
DataOffset: nodeStart, |
|||
Size: nodeStop - nodeStart, |
|||
Next: nil, |
|||
}) |
|||
} |
|||
for i := 1; i < len(nodes); i++ { |
|||
nodes[i-1].Next = nodes[i] |
|||
} |
|||
return &WrittenIntervalLinkedList{ |
|||
tempFile: list.tempFile, |
|||
Head: nodes[0], |
|||
Tail: nodes[len(nodes)-1], |
|||
} |
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) debug() { |
|||
log.Printf("++") |
|||
for _, l := range c.lists { |
|||
log.Printf("++++") |
|||
for t := l.Head; ; t = t.Next { |
|||
log.Printf("[%d,%d) => [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size) |
|||
if t.Next == nil { |
|||
break |
|||
} |
|||
} |
|||
log.Printf("----") |
|||
} |
|||
log.Printf("--") |
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) AddInterval(tempOffset int64, dataSize int, dataOffset int64) { |
|||
|
|||
interval := &WrittenIntervalNode{DataOffset: dataOffset, TempOffset: tempOffset, Size: int64(dataSize)} |
|||
|
|||
// append to the tail and return
|
|||
if len(c.lists) == 1 { |
|||
lastSpan := c.lists[0] |
|||
if lastSpan.Tail.DataOffset+lastSpan.Tail.Size == dataOffset { |
|||
lastSpan.addNodeToTail(interval) |
|||
return |
|||
} |
|||
} |
|||
|
|||
var newLists []*WrittenIntervalLinkedList |
|||
for _, list := range c.lists { |
|||
// if list is to the left of new interval, add to the new list
|
|||
if list.Tail.DataOffset+list.Tail.Size <= interval.DataOffset { |
|||
newLists = append(newLists, list) |
|||
} |
|||
// if list is to the right of new interval, add to the new list
|
|||
if interval.DataOffset+interval.Size <= list.Head.DataOffset { |
|||
newLists = append(newLists, list) |
|||
} |
|||
// if new interval overwrite the right part of the list
|
|||
if list.Head.DataOffset < interval.DataOffset && interval.DataOffset < list.Tail.DataOffset+list.Tail.Size { |
|||
// create a new list of the left part of existing list
|
|||
newLists = append(newLists, list.subList(list.Offset(), interval.DataOffset)) |
|||
} |
|||
// if new interval overwrite the left part of the list
|
|||
if list.Head.DataOffset < interval.DataOffset+interval.Size && interval.DataOffset+interval.Size < list.Tail.DataOffset+list.Tail.Size { |
|||
// create a new list of the right part of existing list
|
|||
newLists = append(newLists, list.subList(interval.DataOffset+interval.Size, list.Tail.DataOffset+list.Tail.Size)) |
|||
} |
|||
// skip anything that is fully overwritten by the new interval
|
|||
} |
|||
|
|||
c.lists = newLists |
|||
// add the new interval to the lists, connecting neighbor lists
|
|||
var prevList, nextList *WrittenIntervalLinkedList |
|||
|
|||
for _, list := range c.lists { |
|||
if list.Head.DataOffset == interval.DataOffset+interval.Size { |
|||
nextList = list |
|||
break |
|||
} |
|||
} |
|||
|
|||
for _, list := range c.lists { |
|||
if list.Head.DataOffset+list.Size() == dataOffset { |
|||
list.addNodeToTail(interval) |
|||
prevList = list |
|||
break |
|||
} |
|||
} |
|||
|
|||
if prevList != nil && nextList != nil { |
|||
// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
|
|||
prevList.Tail.Next = nextList.Head |
|||
prevList.Tail = nextList.Tail |
|||
c.removeList(nextList) |
|||
} else if nextList != nil { |
|||
// add to head was not done when checking
|
|||
nextList.addNodeToHead(interval) |
|||
} |
|||
if prevList == nil && nextList == nil { |
|||
c.lists = append(c.lists, &WrittenIntervalLinkedList{ |
|||
tempFile: c.tempFile, |
|||
Head: interval, |
|||
Tail: interval, |
|||
}) |
|||
} |
|||
|
|||
return |
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) RemoveLargestIntervalLinkedList() *WrittenIntervalLinkedList { |
|||
var maxSize int64 |
|||
maxIndex := -1 |
|||
for k, list := range c.lists { |
|||
if maxSize <= list.Size() { |
|||
maxSize = list.Size() |
|||
maxIndex = k |
|||
} |
|||
} |
|||
if maxSize <= 0 { |
|||
return nil |
|||
} |
|||
|
|||
t := c.lists[maxIndex] |
|||
t.tempFile = c.tempFile |
|||
c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...) |
|||
return t |
|||
|
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) removeList(target *WrittenIntervalLinkedList) { |
|||
index := -1 |
|||
for k, list := range c.lists { |
|||
if list.Offset() == target.Offset() { |
|||
index = k |
|||
} |
|||
} |
|||
if index < 0 { |
|||
return |
|||
} |
|||
|
|||
c.lists = append(c.lists[0:index], c.lists[index+1:]...) |
|||
|
|||
} |
|||
|
|||
func (c *WrittenContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxStop int64) { |
|||
for _, list := range c.lists { |
|||
start := max(startOffset, list.Offset()) |
|||
stop := min(startOffset+int64(len(data)), list.Offset()+list.Size()) |
|||
if start < stop { |
|||
list.ReadData(data[start-startOffset:], start, stop) |
|||
maxStop = max(maxStop, stop) |
|||
} |
|||
} |
|||
return |
|||
} |
|||
|
|||
func (l *WrittenIntervalLinkedList) ToReader(start int64, stop int64) io.Reader { |
|||
// TODO: optimize this to avoid another loop
|
|||
var readers []io.Reader |
|||
for t := l.Head; ; t = t.Next { |
|||
startOffset, stopOffset := max(t.DataOffset, start), min(t.DataOffset+t.Size, stop) |
|||
if startOffset < stopOffset { |
|||
// glog.V(4).Infof("ToReader read [%d,%d) from [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size)
|
|||
readers = append(readers, newFileSectionReader(l.tempFile, startOffset-t.DataOffset+t.TempOffset, startOffset, stopOffset-startOffset)) |
|||
} |
|||
if t.Next == nil { |
|||
break |
|||
} |
|||
} |
|||
if len(readers) == 1 { |
|||
return readers[0] |
|||
} |
|||
return io.MultiReader(readers...) |
|||
} |
|||
|
|||
type FileSectionReader struct { |
|||
file *os.File |
|||
tempStartOffset int64 |
|||
Offset int64 |
|||
dataStart int64 |
|||
dataStop int64 |
|||
} |
|||
|
|||
var _ = io.Reader(&FileSectionReader{}) |
|||
|
|||
func newFileSectionReader(tempfile *os.File, offset int64, dataOffset int64, size int64) *FileSectionReader { |
|||
return &FileSectionReader{ |
|||
file: tempfile, |
|||
tempStartOffset: offset, |
|||
Offset: offset, |
|||
dataStart: dataOffset, |
|||
dataStop: dataOffset + size, |
|||
} |
|||
} |
|||
|
|||
func (f *FileSectionReader) Read(p []byte) (n int, err error) { |
|||
remaining := (f.dataStop - f.dataStart) - (f.Offset - f.tempStartOffset) |
|||
if remaining <= 0 { |
|||
return 0, io.EOF |
|||
} |
|||
dataLen := min(remaining, int64(len(p))) |
|||
// glog.V(4).Infof("reading [%d,%d) from %v [%d,%d)/[%d,%d) %d", f.Offset-f.tempStartOffset+f.dataStart, f.Offset-f.tempStartOffset+f.dataStart+dataLen, f.file.Name(), f.Offset, f.Offset+dataLen, f.tempStartOffset, f.tempStartOffset+f.dataStop-f.dataStart, f.dataStop-f.dataStart)
|
|||
n, err = f.file.ReadAt(p[:dataLen], f.Offset) |
|||
if n > 0 { |
|||
f.Offset += int64(n) |
|||
} else { |
|||
err = io.EOF |
|||
} |
|||
return |
|||
} |
|||
13
weed/statik/statik.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Write
Preview
Loading…
Cancel
Save
Reference in new issue