hilimd
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 870 additions and 100 deletions
-
114.github/workflows/container_latest.yml
-
118.github/workflows/container_release.yml
-
2README.md
-
6docker/Dockerfile.go_build
-
6docker/Dockerfile.go_build_large
-
2docker/compose/local-mount-profile-compose.yml
-
6go.mod
-
2go.sum
-
4k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
62weed/command/filer_copy.go
-
2weed/filer/filerstore_wrapper.go
-
6weed/filer/reader_at.go
-
2weed/filesys/dir.go
-
9weed/filesys/dir_rename.go
-
3weed/filesys/dirty_page_interval.go
-
10weed/filesys/dirty_pages.go
-
32weed/filesys/dirty_pages_continuous.go
-
166weed/filesys/dirty_pages_temp_file.go
-
289weed/filesys/dirty_pages_temp_interval.go
-
10weed/filesys/file.go
-
25weed/filesys/filehandle.go
-
2weed/filesys/meta_cache/meta_cache.go
-
7weed/filesys/wfs.go
-
2weed/s3api/s3err/s3api_errors.go
-
16weed/server/common.go
-
2weed/server/volume_grpc_copy.go
-
13weed/statik/statik.go
-
4weed/storage/volume_vacuum.go
-
4weed/topology/node.go
-
8weed/topology/topology.go
-
11weed/topology/topology_event_handling.go
-
6weed/topology/volume_layout.go
-
2weed/util/bounded_tree/bounded_tree.go
-
2weed/util/constants.go
-
13weed/weed.go
@ -0,0 +1,114 @@ |
|||||
|
name: Build Latest Containers |
||||
|
on: |
||||
|
push: |
||||
|
branches: |
||||
|
- master |
||||
|
workflow_dispatch: [] |
||||
|
|
||||
|
jobs: |
||||
|
build-latest: |
||||
|
runs-on: [ubuntu-latest] |
||||
|
steps: |
||||
|
- |
||||
|
name: Checkout |
||||
|
uses: actions/checkout@v2 |
||||
|
- |
||||
|
name: Docker meta |
||||
|
id: docker_meta |
||||
|
uses: crazy-max/ghaction-docker-meta@v2 |
||||
|
with: |
||||
|
images: | |
||||
|
chrislusf/seaweedfs |
||||
|
ghcr.io/chrislusf/seaweedfs |
||||
|
tags: | |
||||
|
type=raw,value=latest |
||||
|
labels: | |
||||
|
org.opencontainers.image.title=seaweedfs |
||||
|
org.opencontainers.image.vendor=Chris Lu |
||||
|
- |
||||
|
name: Set up QEMU |
||||
|
uses: docker/setup-qemu-action@v1 |
||||
|
- |
||||
|
name: Set up Docker Buildx |
||||
|
uses: docker/setup-buildx-action@v1 |
||||
|
with: |
||||
|
buildkitd-flags: "--debug" |
||||
|
- |
||||
|
name: Login to Docker Hub |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
username: ${{ secrets.DOCKER_USERNAME }} |
||||
|
password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
- |
||||
|
name: Login to GHCR |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
registry: ghcr.io |
||||
|
username: ${{ secrets.GHCR_USERNAME }} |
||||
|
password: ${{ secrets.GHCR_TOKEN }} |
||||
|
- |
||||
|
name: Build |
||||
|
uses: docker/build-push-action@v2 |
||||
|
with: |
||||
|
context: ./docker |
||||
|
push: ${{ github.event_name != 'pull_request' }} |
||||
|
file: ./docker/Dockerfile |
||||
|
platforms: linux/amd64 |
||||
|
tags: ${{ steps.docker_meta.outputs.tags }} |
||||
|
labels: ${{ steps.docker_meta.outputs.labels }} |
||||
|
|
||||
|
build-dev: |
||||
|
runs-on: [ubuntu-latest] |
||||
|
steps: |
||||
|
- |
||||
|
name: Checkout |
||||
|
uses: actions/checkout@v2 |
||||
|
- |
||||
|
name: Docker meta |
||||
|
id: docker_meta |
||||
|
uses: crazy-max/ghaction-docker-meta@v2 |
||||
|
with: |
||||
|
images: | |
||||
|
chrislusf/seaweedfs |
||||
|
ghcr.io/chrislusf/seaweedfs |
||||
|
tags: | |
||||
|
type=raw,value=dev |
||||
|
labels: | |
||||
|
org.opencontainers.image.title=seaweedfs |
||||
|
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
||||
|
org.opencontainers.image.vendor=Chris Lu |
||||
|
- |
||||
|
name: Set up QEMU |
||||
|
uses: docker/setup-qemu-action@v1 |
||||
|
- |
||||
|
name: Set up Docker Buildx |
||||
|
uses: docker/setup-buildx-action@v1 |
||||
|
with: |
||||
|
buildkitd-flags: "--debug" |
||||
|
- |
||||
|
name: Login to Docker Hub |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
username: ${{ secrets.DOCKER_USERNAME }} |
||||
|
password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
- |
||||
|
name: Login to GHCR |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
registry: ghcr.io |
||||
|
username: ${{ secrets.GHCR_USERNAME }} |
||||
|
password: ${{ secrets.GHCR_TOKEN }} |
||||
|
- |
||||
|
name: Build |
||||
|
uses: docker/build-push-action@v2 |
||||
|
with: |
||||
|
context: ./docker |
||||
|
push: ${{ github.event_name != 'pull_request' }} |
||||
|
file: ./docker/Dockerfile.go_build |
||||
|
platforms: linux/amd64 |
||||
|
tags: ${{ steps.docker_meta.outputs.tags }} |
||||
|
labels: ${{ steps.docker_meta.outputs.labels }} |
||||
@ -0,0 +1,118 @@ |
|||||
|
name: Build Release Containers |
||||
|
on: |
||||
|
push: |
||||
|
tags: |
||||
|
- '*' |
||||
|
workflow_dispatch: [] |
||||
|
|
||||
|
jobs: |
||||
|
build-default: |
||||
|
runs-on: [ubuntu-latest] |
||||
|
steps: |
||||
|
- |
||||
|
name: Checkout |
||||
|
uses: actions/checkout@v2 |
||||
|
- |
||||
|
name: Docker meta |
||||
|
id: docker_meta |
||||
|
uses: crazy-max/ghaction-docker-meta@v2 |
||||
|
with: |
||||
|
images: | |
||||
|
chrislusf/seaweedfs |
||||
|
ghcr.io/chrislusf/seaweedfs |
||||
|
tags: | |
||||
|
type=ref,event=tag |
||||
|
flavor: | |
||||
|
latest=false |
||||
|
labels: | |
||||
|
org.opencontainers.image.title=seaweedfs |
||||
|
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
||||
|
org.opencontainers.image.vendor=Chris Lu |
||||
|
- |
||||
|
name: Set up QEMU |
||||
|
uses: docker/setup-qemu-action@v1 |
||||
|
- |
||||
|
name: Set up Docker Buildx |
||||
|
uses: docker/setup-buildx-action@v1 |
||||
|
with: |
||||
|
buildkitd-flags: "--debug" |
||||
|
- |
||||
|
name: Login to Docker Hub |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
username: ${{ secrets.DOCKER_USERNAME }} |
||||
|
password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
- |
||||
|
name: Login to GHCR |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
registry: ghcr.io |
||||
|
username: ${{ secrets.GHCR_USERNAME }} |
||||
|
password: ${{ secrets.GHCR_TOKEN }} |
||||
|
- |
||||
|
name: Build |
||||
|
uses: docker/build-push-action@v2 |
||||
|
with: |
||||
|
context: ./docker |
||||
|
push: ${{ github.event_name != 'pull_request' }} |
||||
|
file: ./docker/Dockerfile.go_build |
||||
|
platforms: linux/amd64 |
||||
|
tags: ${{ steps.docker_meta.outputs.tags }} |
||||
|
labels: ${{ steps.docker_meta.outputs.labels }} |
||||
|
build-large: |
||||
|
runs-on: [ubuntu-latest] |
||||
|
steps: |
||||
|
- |
||||
|
name: Checkout |
||||
|
uses: actions/checkout@v2 |
||||
|
- |
||||
|
name: Docker meta |
||||
|
id: docker_meta |
||||
|
uses: crazy-max/ghaction-docker-meta@v2 |
||||
|
with: |
||||
|
images: | |
||||
|
chrislusf/seaweedfs |
||||
|
ghcr.io/chrislusf/seaweedfs |
||||
|
tags: | |
||||
|
type=ref,event=tag,suffix=_large_disk |
||||
|
flavor: | |
||||
|
latest=false |
||||
|
labels: | |
||||
|
org.opencontainers.image.title=seaweedfs |
||||
|
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! |
||||
|
org.opencontainers.image.vendor=Chris Lu |
||||
|
- |
||||
|
name: Set up QEMU |
||||
|
uses: docker/setup-qemu-action@v1 |
||||
|
- |
||||
|
name: Set up Docker Buildx |
||||
|
uses: docker/setup-buildx-action@v1 |
||||
|
with: |
||||
|
buildkitd-flags: "--debug" |
||||
|
- |
||||
|
name: Login to Docker Hub |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
username: ${{ secrets.DOCKER_USERNAME }} |
||||
|
password: ${{ secrets.DOCKER_PASSWORD }} |
||||
|
- |
||||
|
name: Login to GHCR |
||||
|
if: github.event_name != 'pull_request' |
||||
|
uses: docker/login-action@v1 |
||||
|
with: |
||||
|
registry: ghcr.io |
||||
|
username: ${{ secrets.GHCR_USERNAME }} |
||||
|
password: ${{ secrets.GHCR_TOKEN }} |
||||
|
- |
||||
|
name: Build |
||||
|
uses: docker/build-push-action@v2 |
||||
|
with: |
||||
|
context: ./docker |
||||
|
push: ${{ github.event_name != 'pull_request' }} |
||||
|
file: ./docker/Dockerfile.go_build_large |
||||
|
platforms: linux/amd64 |
||||
|
tags: ${{ steps.docker_meta.outputs.tags }} |
||||
|
labels: ${{ steps.docker_meta.outputs.labels }} |
||||
@ -1,5 +1,5 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
appVersion: "2.43" |
|
||||
version: 2.43 |
|
||||
|
appVersion: "2.48" |
||||
|
version: 2.48 |
||||
@ -0,0 +1,10 @@ |
|||||
|
package filesys |
||||
|
|
||||
|
type DirtyPages interface { |
||||
|
AddPage(offset int64, data []byte) |
||||
|
FlushData() error |
||||
|
ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) |
||||
|
GetStorageOptions() (collection, replication string) |
||||
|
SetWriteOnly(writeOnly bool) |
||||
|
GetWriteOnly() (writeOnly bool) |
||||
|
} |
||||
@ -0,0 +1,166 @@ |
|||||
|
package filesys |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"io" |
||||
|
"os" |
||||
|
"path/filepath" |
||||
|
"sync" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type TempFileDirtyPages struct { |
||||
|
f *File |
||||
|
tf *os.File |
||||
|
writtenIntervals *WrittenContinuousIntervals |
||||
|
writeOnly bool |
||||
|
writeWaitGroup sync.WaitGroup |
||||
|
pageAddLock sync.Mutex |
||||
|
chunkAddLock sync.Mutex |
||||
|
lastErr error |
||||
|
collection string |
||||
|
replication string |
||||
|
} |
||||
|
|
||||
|
var ( |
||||
|
tmpDir = filepath.Join(os.TempDir(), "sw") |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
os.Mkdir(tmpDir, 0755) |
||||
|
} |
||||
|
|
||||
|
func newTempFileDirtyPages(file *File, writeOnly bool) *TempFileDirtyPages { |
||||
|
|
||||
|
tempFile := &TempFileDirtyPages{ |
||||
|
f: file, |
||||
|
writeOnly: writeOnly, |
||||
|
writtenIntervals: &WrittenContinuousIntervals{}, |
||||
|
} |
||||
|
|
||||
|
return tempFile |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) AddPage(offset int64, data []byte) { |
||||
|
|
||||
|
pages.pageAddLock.Lock() |
||||
|
defer pages.pageAddLock.Unlock() |
||||
|
|
||||
|
if pages.tf == nil { |
||||
|
tf, err := os.CreateTemp(tmpDir, "") |
||||
|
if err != nil { |
||||
|
glog.Errorf("create temp file: %v", err) |
||||
|
pages.lastErr = err |
||||
|
return |
||||
|
} |
||||
|
pages.tf = tf |
||||
|
pages.writtenIntervals.tempFile = tf |
||||
|
pages.writtenIntervals.lastOffset = 0 |
||||
|
} |
||||
|
|
||||
|
writtenOffset := pages.writtenIntervals.lastOffset |
||||
|
dataSize := int64(len(data)) |
||||
|
|
||||
|
// glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+dataSize)
|
||||
|
|
||||
|
if _, err := pages.tf.WriteAt(data, writtenOffset); err != nil { |
||||
|
pages.lastErr = err |
||||
|
} else { |
||||
|
pages.writtenIntervals.AddInterval(writtenOffset, len(data), offset) |
||||
|
pages.writtenIntervals.lastOffset += dataSize |
||||
|
} |
||||
|
|
||||
|
// pages.writtenIntervals.debug()
|
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) FlushData() error { |
||||
|
|
||||
|
pages.saveExistingPagesToStorage() |
||||
|
pages.writeWaitGroup.Wait() |
||||
|
if pages.lastErr != nil { |
||||
|
return fmt.Errorf("flush data: %v", pages.lastErr) |
||||
|
} |
||||
|
pages.pageAddLock.Lock() |
||||
|
defer pages.pageAddLock.Unlock() |
||||
|
if pages.tf != nil { |
||||
|
|
||||
|
pages.writtenIntervals.tempFile = nil |
||||
|
pages.writtenIntervals.lists = nil |
||||
|
|
||||
|
pages.tf.Close() |
||||
|
os.Remove(pages.tf.Name()) |
||||
|
pages.tf = nil |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) saveExistingPagesToStorage() { |
||||
|
|
||||
|
pageSize := pages.f.wfs.option.ChunkSizeLimit |
||||
|
|
||||
|
// glog.V(4).Infof("%v saveExistingPagesToStorage %d lists", pages.f.Name, len(pages.writtenIntervals.lists))
|
||||
|
|
||||
|
for _, list := range pages.writtenIntervals.lists { |
||||
|
listStopOffset := list.Offset() + list.Size() |
||||
|
for uploadedOffset:=int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize { |
||||
|
start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize) |
||||
|
if start >= stop { |
||||
|
continue |
||||
|
} |
||||
|
// glog.V(4).Infof("uploading %v [%d,%d) %d/%d", pages.f.Name, start, stop, i, len(pages.writtenIntervals.lists))
|
||||
|
pages.saveToStorage(list.ToReader(start, stop), start, stop-start) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) { |
||||
|
|
||||
|
mtime := time.Now().UnixNano() |
||||
|
pages.writeWaitGroup.Add(1) |
||||
|
writer := func() { |
||||
|
defer pages.writeWaitGroup.Done() |
||||
|
|
||||
|
reader = io.LimitReader(reader, size) |
||||
|
chunk, collection, replication, err := pages.f.wfs.saveDataAsChunk(pages.f.fullpath(), pages.writeOnly)(reader, pages.f.Name, offset) |
||||
|
if err != nil { |
||||
|
glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), offset, offset+size, err) |
||||
|
pages.lastErr = err |
||||
|
return |
||||
|
} |
||||
|
chunk.Mtime = mtime |
||||
|
pages.collection, pages.replication = collection, replication |
||||
|
pages.chunkAddLock.Lock() |
||||
|
defer pages.chunkAddLock.Unlock() |
||||
|
pages.f.addChunks([]*filer_pb.FileChunk{chunk}) |
||||
|
glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.f.fullpath(), chunk.FileId, offset, offset+size) |
||||
|
} |
||||
|
|
||||
|
if pages.f.wfs.concurrentWriters != nil { |
||||
|
pages.f.wfs.concurrentWriters.Execute(writer) |
||||
|
} else { |
||||
|
go writer() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { |
||||
|
return pages.writtenIntervals.ReadDataAt(data, startOffset) |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) GetStorageOptions() (collection, replication string) { |
||||
|
return pages.collection, pages.replication |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) SetWriteOnly(writeOnly bool) { |
||||
|
if pages.writeOnly { |
||||
|
pages.writeOnly = writeOnly |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (pages *TempFileDirtyPages) GetWriteOnly() (writeOnly bool) { |
||||
|
return pages.writeOnly |
||||
|
} |
||||
@ -0,0 +1,289 @@ |
|||||
|
package filesys |
||||
|
|
||||
|
import ( |
||||
|
"io" |
||||
|
"log" |
||||
|
"os" |
||||
|
) |
||||
|
|
||||
|
type WrittenIntervalNode struct { |
||||
|
DataOffset int64 |
||||
|
TempOffset int64 |
||||
|
Size int64 |
||||
|
Next *WrittenIntervalNode |
||||
|
} |
||||
|
|
||||
|
type WrittenIntervalLinkedList struct { |
||||
|
tempFile *os.File |
||||
|
Head *WrittenIntervalNode |
||||
|
Tail *WrittenIntervalNode |
||||
|
} |
||||
|
|
||||
|
type WrittenContinuousIntervals struct { |
||||
|
tempFile *os.File |
||||
|
lastOffset int64 |
||||
|
lists []*WrittenIntervalLinkedList |
||||
|
} |
||||
|
|
||||
|
func (list *WrittenIntervalLinkedList) Offset() int64 { |
||||
|
return list.Head.DataOffset |
||||
|
} |
||||
|
func (list *WrittenIntervalLinkedList) Size() int64 { |
||||
|
return list.Tail.DataOffset + list.Tail.Size - list.Head.DataOffset |
||||
|
} |
||||
|
func (list *WrittenIntervalLinkedList) addNodeToTail(node *WrittenIntervalNode) { |
||||
|
// glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
|
||||
|
if list.Tail.TempOffset+list.Tail.Size == node.TempOffset { |
||||
|
// already connected
|
||||
|
list.Tail.Size += node.Size |
||||
|
} else { |
||||
|
list.Tail.Next = node |
||||
|
list.Tail = node |
||||
|
} |
||||
|
} |
||||
|
func (list *WrittenIntervalLinkedList) addNodeToHead(node *WrittenIntervalNode) { |
||||
|
// glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
|
||||
|
node.Next = list.Head |
||||
|
list.Head = node |
||||
|
} |
||||
|
|
||||
|
func (list *WrittenIntervalLinkedList) ReadData(buf []byte, start, stop int64) { |
||||
|
t := list.Head |
||||
|
for { |
||||
|
|
||||
|
nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) |
||||
|
if nodeStart < nodeStop { |
||||
|
// glog.V(4).Infof("copying start=%d stop=%d t=[%d,%d) => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.DataOffset, t.DataOffset+t.Size, len(buf), nodeStart, nodeStop)
|
||||
|
list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset + nodeStart - t.DataOffset) |
||||
|
} |
||||
|
|
||||
|
if t.Next == nil { |
||||
|
break |
||||
|
} |
||||
|
t = t.Next |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) TotalSize() (total int64) { |
||||
|
for _, list := range c.lists { |
||||
|
total += list.Size() |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (list *WrittenIntervalLinkedList) subList(start, stop int64) *WrittenIntervalLinkedList { |
||||
|
var nodes []*WrittenIntervalNode |
||||
|
for t := list.Head; t != nil; t = t.Next { |
||||
|
nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size) |
||||
|
if nodeStart >= nodeStop { |
||||
|
// skip non overlapping WrittenIntervalNode
|
||||
|
continue |
||||
|
} |
||||
|
nodes = append(nodes, &WrittenIntervalNode{ |
||||
|
TempOffset: t.TempOffset + nodeStart - t.DataOffset, |
||||
|
DataOffset: nodeStart, |
||||
|
Size: nodeStop - nodeStart, |
||||
|
Next: nil, |
||||
|
}) |
||||
|
} |
||||
|
for i := 1; i < len(nodes); i++ { |
||||
|
nodes[i-1].Next = nodes[i] |
||||
|
} |
||||
|
return &WrittenIntervalLinkedList{ |
||||
|
tempFile: list.tempFile, |
||||
|
Head: nodes[0], |
||||
|
Tail: nodes[len(nodes)-1], |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) debug() { |
||||
|
log.Printf("++") |
||||
|
for _, l := range c.lists { |
||||
|
log.Printf("++++") |
||||
|
for t := l.Head; ; t = t.Next { |
||||
|
log.Printf("[%d,%d) => [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size) |
||||
|
if t.Next == nil { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
log.Printf("----") |
||||
|
} |
||||
|
log.Printf("--") |
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) AddInterval(tempOffset int64, dataSize int, dataOffset int64) { |
||||
|
|
||||
|
interval := &WrittenIntervalNode{DataOffset: dataOffset, TempOffset: tempOffset, Size: int64(dataSize)} |
||||
|
|
||||
|
// append to the tail and return
|
||||
|
if len(c.lists) == 1 { |
||||
|
lastSpan := c.lists[0] |
||||
|
if lastSpan.Tail.DataOffset+lastSpan.Tail.Size == dataOffset { |
||||
|
lastSpan.addNodeToTail(interval) |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
var newLists []*WrittenIntervalLinkedList |
||||
|
for _, list := range c.lists { |
||||
|
// if list is to the left of new interval, add to the new list
|
||||
|
if list.Tail.DataOffset+list.Tail.Size <= interval.DataOffset { |
||||
|
newLists = append(newLists, list) |
||||
|
} |
||||
|
// if list is to the right of new interval, add to the new list
|
||||
|
if interval.DataOffset+interval.Size <= list.Head.DataOffset { |
||||
|
newLists = append(newLists, list) |
||||
|
} |
||||
|
// if new interval overwrite the right part of the list
|
||||
|
if list.Head.DataOffset < interval.DataOffset && interval.DataOffset < list.Tail.DataOffset+list.Tail.Size { |
||||
|
// create a new list of the left part of existing list
|
||||
|
newLists = append(newLists, list.subList(list.Offset(), interval.DataOffset)) |
||||
|
} |
||||
|
// if new interval overwrite the left part of the list
|
||||
|
if list.Head.DataOffset < interval.DataOffset+interval.Size && interval.DataOffset+interval.Size < list.Tail.DataOffset+list.Tail.Size { |
||||
|
// create a new list of the right part of existing list
|
||||
|
newLists = append(newLists, list.subList(interval.DataOffset+interval.Size, list.Tail.DataOffset+list.Tail.Size)) |
||||
|
} |
||||
|
// skip anything that is fully overwritten by the new interval
|
||||
|
} |
||||
|
|
||||
|
c.lists = newLists |
||||
|
// add the new interval to the lists, connecting neighbor lists
|
||||
|
var prevList, nextList *WrittenIntervalLinkedList |
||||
|
|
||||
|
for _, list := range c.lists { |
||||
|
if list.Head.DataOffset == interval.DataOffset+interval.Size { |
||||
|
nextList = list |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for _, list := range c.lists { |
||||
|
if list.Head.DataOffset+list.Size() == dataOffset { |
||||
|
list.addNodeToTail(interval) |
||||
|
prevList = list |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if prevList != nil && nextList != nil { |
||||
|
// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
|
||||
|
prevList.Tail.Next = nextList.Head |
||||
|
prevList.Tail = nextList.Tail |
||||
|
c.removeList(nextList) |
||||
|
} else if nextList != nil { |
||||
|
// add to head was not done when checking
|
||||
|
nextList.addNodeToHead(interval) |
||||
|
} |
||||
|
if prevList == nil && nextList == nil { |
||||
|
c.lists = append(c.lists, &WrittenIntervalLinkedList{ |
||||
|
tempFile: c.tempFile, |
||||
|
Head: interval, |
||||
|
Tail: interval, |
||||
|
}) |
||||
|
} |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) RemoveLargestIntervalLinkedList() *WrittenIntervalLinkedList { |
||||
|
var maxSize int64 |
||||
|
maxIndex := -1 |
||||
|
for k, list := range c.lists { |
||||
|
if maxSize <= list.Size() { |
||||
|
maxSize = list.Size() |
||||
|
maxIndex = k |
||||
|
} |
||||
|
} |
||||
|
if maxSize <= 0 { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
t := c.lists[maxIndex] |
||||
|
t.tempFile = c.tempFile |
||||
|
c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...) |
||||
|
return t |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) removeList(target *WrittenIntervalLinkedList) { |
||||
|
index := -1 |
||||
|
for k, list := range c.lists { |
||||
|
if list.Offset() == target.Offset() { |
||||
|
index = k |
||||
|
} |
||||
|
} |
||||
|
if index < 0 { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
c.lists = append(c.lists[0:index], c.lists[index+1:]...) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func (c *WrittenContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxStop int64) { |
||||
|
for _, list := range c.lists { |
||||
|
start := max(startOffset, list.Offset()) |
||||
|
stop := min(startOffset+int64(len(data)), list.Offset()+list.Size()) |
||||
|
if start < stop { |
||||
|
list.ReadData(data[start-startOffset:], start, stop) |
||||
|
maxStop = max(maxStop, stop) |
||||
|
} |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (l *WrittenIntervalLinkedList) ToReader(start int64, stop int64) io.Reader { |
||||
|
// TODO: optimize this to avoid another loop
|
||||
|
var readers []io.Reader |
||||
|
for t := l.Head; ; t = t.Next { |
||||
|
startOffset, stopOffset := max(t.DataOffset, start), min(t.DataOffset+t.Size, stop) |
||||
|
if startOffset < stopOffset { |
||||
|
// glog.V(4).Infof("ToReader read [%d,%d) from [%d,%d) %d", t.DataOffset, t.DataOffset+t.Size, t.TempOffset, t.TempOffset+t.Size, t.Size)
|
||||
|
readers = append(readers, newFileSectionReader(l.tempFile, startOffset-t.DataOffset+t.TempOffset, startOffset, stopOffset-startOffset)) |
||||
|
} |
||||
|
if t.Next == nil { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
if len(readers) == 1 { |
||||
|
return readers[0] |
||||
|
} |
||||
|
return io.MultiReader(readers...) |
||||
|
} |
||||
|
|
||||
|
type FileSectionReader struct { |
||||
|
file *os.File |
||||
|
tempStartOffset int64 |
||||
|
Offset int64 |
||||
|
dataStart int64 |
||||
|
dataStop int64 |
||||
|
} |
||||
|
|
||||
|
var _ = io.Reader(&FileSectionReader{}) |
||||
|
|
||||
|
func newFileSectionReader(tempfile *os.File, offset int64, dataOffset int64, size int64) *FileSectionReader { |
||||
|
return &FileSectionReader{ |
||||
|
file: tempfile, |
||||
|
tempStartOffset: offset, |
||||
|
Offset: offset, |
||||
|
dataStart: dataOffset, |
||||
|
dataStop: dataOffset + size, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (f *FileSectionReader) Read(p []byte) (n int, err error) { |
||||
|
remaining := (f.dataStop - f.dataStart) - (f.Offset - f.tempStartOffset) |
||||
|
if remaining <= 0 { |
||||
|
return 0, io.EOF |
||||
|
} |
||||
|
dataLen := min(remaining, int64(len(p))) |
||||
|
// glog.V(4).Infof("reading [%d,%d) from %v [%d,%d)/[%d,%d) %d", f.Offset-f.tempStartOffset+f.dataStart, f.Offset-f.tempStartOffset+f.dataStart+dataLen, f.file.Name(), f.Offset, f.Offset+dataLen, f.tempStartOffset, f.tempStartOffset+f.dataStop-f.dataStart, f.dataStop-f.dataStart)
|
||||
|
n, err = f.file.ReadAt(p[:dataLen], f.Offset) |
||||
|
if n > 0 { |
||||
|
f.Offset += int64(n) |
||||
|
} else { |
||||
|
err = io.EOF |
||||
|
} |
||||
|
return |
||||
|
} |
||||
13
weed/statik/statik.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Write
Preview
Loading…
Cancel
Save
Reference in new issue