yourchanges
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
78 changed files with 1548 additions and 2415 deletions
-
2README.md
-
2k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
170other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
22other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
-
11other/java/client/src/main/java/seaweedfs/client/ChunkCache.java
-
5other/java/client/src/main/java/seaweedfs/client/FilerClient.java
-
4other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
-
41other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
-
27other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java
-
20other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java
-
5other/java/client/src/main/proto/filer.proto
-
2other/java/hdfs2/dependency-reduced-pom.xml
-
2other/java/hdfs2/pom.xml
-
137other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBuffer.java
-
394other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBufferManager.java
-
70other/java/hdfs2/src/main/java/seaweed/hdfs/ReadBufferWorker.java
-
22other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedAbstractFileSystem.java
-
28other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
-
12other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
183other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedInputStream.java
-
89other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedOutputStream.java
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
2other/java/hdfs3/pom.xml
-
137other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBuffer.java
-
394other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBufferManager.java
-
70other/java/hdfs3/src/main/java/seaweed/hdfs/ReadBufferWorker.java
-
22other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedAbstractFileSystem.java
-
26other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
-
12other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
-
183other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java
-
98other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java
-
0test/data/sample.idx
-
111test/s3/basic/basic_test.go
-
4weed/command/backup.go
-
3weed/command/compact.go
-
2weed/command/download.go
-
4weed/command/export.go
-
2weed/command/filer.go
-
5weed/command/fix.go
-
31weed/command/master.go
-
10weed/command/mount_std.go
-
2weed/command/server.go
-
2weed/command/upload.go
-
2weed/command/volume.go
-
2weed/command/webdav.go
-
43weed/filer2/filer.go
-
11weed/filer2/filer_delete_entry.go
-
9weed/filer2/filer_notify.go
-
57weed/filer2/filerstore.go
-
43weed/filer2/leveldb2/leveldb2_local_store.go
-
46weed/filer2/meta_aggregator.go
-
37weed/filer2/meta_replay.go
-
2weed/filesys/meta_cache/meta_cache.go
-
12weed/filesys/meta_cache/meta_cache_init.go
-
4weed/filesys/meta_cache/meta_cache_subscribe.go
-
2weed/filesys/unimplemented.go
-
20weed/filesys/wfs.go
-
5weed/operation/needle_parse_test.go
-
8weed/operation/upload_content.go
-
5weed/pb/filer.proto
-
1162weed/pb/filer_pb/filer.pb.go
-
8weed/s3api/auth_credentials.go
-
4weed/server/common.go
-
23weed/server/filer_grpc_server_sub_meta.go
-
12weed/server/filer_server.go
-
12weed/server/master_server.go
-
2weed/storage/erasure_coding/ec_decoder.go
-
4weed/storage/erasure_coding/ec_volume_test.go
-
2weed/storage/needle_map/compact_map_perf_test.go
-
9weed/storage/store.go
-
5weed/util/bounded_tree/bounded_tree_test.go
-
1weed/util/compression.go
-
2weed/util/constants.go
-
20weed/util/file_util.go
-
13weed/util/log_buffer/log_buffer.go
@ -1,4 +1,4 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
version: 1.84 |
|
||||
|
version: 1.85 |
||||
@ -0,0 +1,170 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?> |
||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
|
||||
|
<groupId>com.github.chrislusf</groupId> |
||||
|
<artifactId>seaweedfs-client</artifactId> |
||||
|
<version>1.3.6</version> |
||||
|
|
||||
|
<parent> |
||||
|
<groupId>org.sonatype.oss</groupId> |
||||
|
<artifactId>oss-parent</artifactId> |
||||
|
<version>9</version> |
||||
|
</parent> |
||||
|
|
||||
|
<properties> |
||||
|
<protobuf.version>3.9.1</protobuf.version> |
||||
|
<!-- follow https://github.com/grpc/grpc-java --> |
||||
|
<grpc.version>1.23.0</grpc.version> |
||||
|
<guava.version>28.0-jre</guava.version> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencies> |
||||
|
<dependency> |
||||
|
<groupId>com.moandjiezana.toml</groupId> |
||||
|
<artifactId>toml4j</artifactId> |
||||
|
<version>0.7.2</version> |
||||
|
</dependency> |
||||
|
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java --> |
||||
|
<dependency> |
||||
|
<groupId>com.google.protobuf</groupId> |
||||
|
<artifactId>protobuf-java</artifactId> |
||||
|
<version>${protobuf.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.google.guava</groupId> |
||||
|
<artifactId>guava</artifactId> |
||||
|
<version>${guava.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-netty-shaded</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-protobuf</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>io.grpc</groupId> |
||||
|
<artifactId>grpc-stub</artifactId> |
||||
|
<version>${grpc.version}</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.slf4j</groupId> |
||||
|
<artifactId>slf4j-api</artifactId> |
||||
|
<version>1.7.25</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.apache.httpcomponents</groupId> |
||||
|
<artifactId>httpmime</artifactId> |
||||
|
<version>4.5.6</version> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>junit</groupId> |
||||
|
<artifactId>junit</artifactId> |
||||
|
<version>4.12</version> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<distributionManagement> |
||||
|
<snapshotRepository> |
||||
|
<id>ossrh</id> |
||||
|
<url>https://oss.sonatype.org/content/repositories/snapshots</url> |
||||
|
</snapshotRepository> |
||||
|
</distributionManagement> |
||||
|
<build> |
||||
|
<extensions> |
||||
|
<extension> |
||||
|
<groupId>kr.motd.maven</groupId> |
||||
|
<artifactId>os-maven-plugin</artifactId> |
||||
|
<version>1.6.2</version> |
||||
|
</extension> |
||||
|
</extensions> |
||||
|
<plugins> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-compiler-plugin</artifactId> |
||||
|
<configuration> |
||||
|
<source>8</source> |
||||
|
<target>8</target> |
||||
|
</configuration> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.xolstice.maven.plugins</groupId> |
||||
|
<artifactId>protobuf-maven-plugin</artifactId> |
||||
|
<version>0.6.1</version> |
||||
|
<configuration> |
||||
|
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} |
||||
|
</protocArtifact> |
||||
|
<pluginId>grpc-java</pluginId> |
||||
|
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} |
||||
|
</pluginArtifact> |
||||
|
</configuration> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<goals> |
||||
|
<goal>compile</goal> |
||||
|
<goal>compile-custom</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-gpg-plugin</artifactId> |
||||
|
<version>1.5</version> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>sign-artifacts</id> |
||||
|
<phase>verify</phase> |
||||
|
<goals> |
||||
|
<goal>sign</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.sonatype.plugins</groupId> |
||||
|
<artifactId>nexus-staging-maven-plugin</artifactId> |
||||
|
<version>1.6.7</version> |
||||
|
<extensions>true</extensions> |
||||
|
<configuration> |
||||
|
<serverId>ossrh</serverId> |
||||
|
<nexusUrl>https://oss.sonatype.org/</nexusUrl> |
||||
|
<autoReleaseAfterClose>true</autoReleaseAfterClose> |
||||
|
</configuration> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-source-plugin</artifactId> |
||||
|
<version>2.2.1</version> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-sources</id> |
||||
|
<goals> |
||||
|
<goal>jar-no-fork</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
<plugin> |
||||
|
<groupId>org.apache.maven.plugins</groupId> |
||||
|
<artifactId>maven-javadoc-plugin</artifactId> |
||||
|
<version>2.9.1</version> |
||||
|
<executions> |
||||
|
<execution> |
||||
|
<id>attach-javadocs</id> |
||||
|
<goals> |
||||
|
<goal>jar</goal> |
||||
|
</goals> |
||||
|
</execution> |
||||
|
</executions> |
||||
|
</plugin> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
|
||||
|
</project> |
||||
@ -0,0 +1,22 @@ |
|||||
|
package seaweedfs.client; |
||||
|
|
||||
|
import java.nio.ByteBuffer; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
public class ByteBufferPool { |
||||
|
|
||||
|
static List<ByteBuffer> bufferList = new ArrayList<>(); |
||||
|
|
||||
|
public static synchronized ByteBuffer request(int bufferSize) { |
||||
|
if (bufferList.isEmpty()) { |
||||
|
return ByteBuffer.allocate(bufferSize); |
||||
|
} |
||||
|
return bufferList.remove(bufferList.size()-1); |
||||
|
} |
||||
|
|
||||
|
public static synchronized void release(ByteBuffer obj) { |
||||
|
bufferList.add(obj); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,27 @@ |
|||||
|
package seaweedfs.client; |
||||
|
|
||||
|
import org.apache.http.impl.DefaultConnectionReuseStrategy; |
||||
|
import org.apache.http.impl.client.CloseableHttpClient; |
||||
|
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; |
||||
|
import org.apache.http.impl.client.HttpClientBuilder; |
||||
|
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; |
||||
|
|
||||
|
public class SeaweedUtil { |
||||
|
|
||||
|
static PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); |
||||
|
|
||||
|
static { |
||||
|
// Increase max total connection to 200 |
||||
|
cm.setMaxTotal(200); |
||||
|
// Increase default max connection per route to 20 |
||||
|
cm.setDefaultMaxPerRoute(20); |
||||
|
} |
||||
|
|
||||
|
public static CloseableHttpClient getClosableHttpClient() { |
||||
|
return HttpClientBuilder.create() |
||||
|
.setConnectionManager(cm) |
||||
|
.setConnectionReuseStrategy(DefaultConnectionReuseStrategy.INSTANCE) |
||||
|
.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE) |
||||
|
.build(); |
||||
|
} |
||||
|
} |
||||
@ -1,137 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
|
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
class ReadBuffer { |
|
||||
|
|
||||
private SeaweedInputStream stream; |
|
||||
private long offset; // offset within the file for the buffer |
|
||||
private int length; // actual length, set after the buffer is filles |
|
||||
private int requestedLength; // requested length of the read |
|
||||
private byte[] buffer; // the buffer itself |
|
||||
private int bufferindex = -1; // index in the buffers array in Buffer manager |
|
||||
private ReadBufferStatus status; // status of the buffer |
|
||||
private CountDownLatch latch = null; // signaled when the buffer is done reading, so any client |
|
||||
// waiting on this buffer gets unblocked |
|
||||
|
|
||||
// fields to help with eviction logic |
|
||||
private long timeStamp = 0; // tick at which buffer became available to read |
|
||||
private boolean isFirstByteConsumed = false; |
|
||||
private boolean isLastByteConsumed = false; |
|
||||
private boolean isAnyByteConsumed = false; |
|
||||
|
|
||||
public SeaweedInputStream getStream() { |
|
||||
return stream; |
|
||||
} |
|
||||
|
|
||||
public void setStream(SeaweedInputStream stream) { |
|
||||
this.stream = stream; |
|
||||
} |
|
||||
|
|
||||
public long getOffset() { |
|
||||
return offset; |
|
||||
} |
|
||||
|
|
||||
public void setOffset(long offset) { |
|
||||
this.offset = offset; |
|
||||
} |
|
||||
|
|
||||
public int getLength() { |
|
||||
return length; |
|
||||
} |
|
||||
|
|
||||
public void setLength(int length) { |
|
||||
this.length = length; |
|
||||
} |
|
||||
|
|
||||
public int getRequestedLength() { |
|
||||
return requestedLength; |
|
||||
} |
|
||||
|
|
||||
public void setRequestedLength(int requestedLength) { |
|
||||
this.requestedLength = requestedLength; |
|
||||
} |
|
||||
|
|
||||
public byte[] getBuffer() { |
|
||||
return buffer; |
|
||||
} |
|
||||
|
|
||||
public void setBuffer(byte[] buffer) { |
|
||||
this.buffer = buffer; |
|
||||
} |
|
||||
|
|
||||
public int getBufferindex() { |
|
||||
return bufferindex; |
|
||||
} |
|
||||
|
|
||||
public void setBufferindex(int bufferindex) { |
|
||||
this.bufferindex = bufferindex; |
|
||||
} |
|
||||
|
|
||||
public ReadBufferStatus getStatus() { |
|
||||
return status; |
|
||||
} |
|
||||
|
|
||||
public void setStatus(ReadBufferStatus status) { |
|
||||
this.status = status; |
|
||||
} |
|
||||
|
|
||||
public CountDownLatch getLatch() { |
|
||||
return latch; |
|
||||
} |
|
||||
|
|
||||
public void setLatch(CountDownLatch latch) { |
|
||||
this.latch = latch; |
|
||||
} |
|
||||
|
|
||||
public long getTimeStamp() { |
|
||||
return timeStamp; |
|
||||
} |
|
||||
|
|
||||
public void setTimeStamp(long timeStamp) { |
|
||||
this.timeStamp = timeStamp; |
|
||||
} |
|
||||
|
|
||||
public boolean isFirstByteConsumed() { |
|
||||
return isFirstByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setFirstByteConsumed(boolean isFirstByteConsumed) { |
|
||||
this.isFirstByteConsumed = isFirstByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public boolean isLastByteConsumed() { |
|
||||
return isLastByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setLastByteConsumed(boolean isLastByteConsumed) { |
|
||||
this.isLastByteConsumed = isLastByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public boolean isAnyByteConsumed() { |
|
||||
return isAnyByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setAnyByteConsumed(boolean isAnyByteConsumed) { |
|
||||
this.isAnyByteConsumed = isAnyByteConsumed; |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,394 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* <p> |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* <p> |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.Collection; |
|
||||
import java.util.LinkedList; |
|
||||
import java.util.Queue; |
|
||||
import java.util.Stack; |
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
/** |
|
||||
* The Read Buffer Manager for Rest AbfsClient. |
|
||||
*/ |
|
||||
final class ReadBufferManager { |
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class); |
|
||||
|
|
||||
private static final int NUM_BUFFERS = 16; |
|
||||
private static final int BLOCK_SIZE = 4 * 1024 * 1024; |
|
||||
private static final int NUM_THREADS = 8; |
|
||||
private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold |
|
||||
|
|
||||
private Thread[] threads = new Thread[NUM_THREADS]; |
|
||||
private byte[][] buffers; // array of byte[] buffers, to hold the data that is read |
|
||||
private Stack<Integer> freeList = new Stack<>(); // indices in buffers[] array that are available |
|
||||
|
|
||||
private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet |
|
||||
private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads |
|
||||
private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading |
|
||||
private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block |
|
||||
|
|
||||
static { |
|
||||
BUFFER_MANAGER = new ReadBufferManager(); |
|
||||
BUFFER_MANAGER.init(); |
|
||||
} |
|
||||
|
|
||||
static ReadBufferManager getBufferManager() { |
|
||||
return BUFFER_MANAGER; |
|
||||
} |
|
||||
|
|
||||
private void init() { |
|
||||
buffers = new byte[NUM_BUFFERS][]; |
|
||||
for (int i = 0; i < NUM_BUFFERS; i++) { |
|
||||
buffers[i] = new byte[BLOCK_SIZE]; // same buffers are reused. The byte array never goes back to GC |
|
||||
freeList.add(i); |
|
||||
} |
|
||||
for (int i = 0; i < NUM_THREADS; i++) { |
|
||||
Thread t = new Thread(new ReadBufferWorker(i)); |
|
||||
t.setDaemon(true); |
|
||||
threads[i] = t; |
|
||||
t.setName("SeaweedFS-prefetch-" + i); |
|
||||
t.start(); |
|
||||
} |
|
||||
ReadBufferWorker.UNLEASH_WORKERS.countDown(); |
|
||||
} |
|
||||
|
|
||||
// hide instance constructor |
|
||||
private ReadBufferManager() { |
|
||||
} |
|
||||
|
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* SeaweedInputStream-facing methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* {@link SeaweedInputStream} calls this method to queue read-aheads. |
|
||||
* |
|
||||
* @param stream The {@link SeaweedInputStream} for which to do the read-ahead |
|
||||
* @param requestedOffset The offset in the file which shoukd be read |
|
||||
* @param requestedLength The length to read |
|
||||
*/ |
|
||||
void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Start Queueing readAhead for {} offset {} length {}", |
|
||||
stream.getPath(), requestedOffset, requestedLength); |
|
||||
} |
|
||||
ReadBuffer buffer; |
|
||||
synchronized (this) { |
|
||||
if (isAlreadyQueued(stream, requestedOffset)) { |
|
||||
return; // already queued, do not queue again |
|
||||
} |
|
||||
if (freeList.isEmpty() && !tryEvict()) { |
|
||||
return; // no buffers available, cannot queue anything |
|
||||
} |
|
||||
|
|
||||
buffer = new ReadBuffer(); |
|
||||
buffer.setStream(stream); |
|
||||
buffer.setOffset(requestedOffset); |
|
||||
buffer.setLength(0); |
|
||||
buffer.setRequestedLength(requestedLength); |
|
||||
buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); |
|
||||
buffer.setLatch(new CountDownLatch(1)); |
|
||||
|
|
||||
Integer bufferIndex = freeList.pop(); // will return a value, since we have checked size > 0 already |
|
||||
|
|
||||
buffer.setBuffer(buffers[bufferIndex]); |
|
||||
buffer.setBufferindex(bufferIndex); |
|
||||
readAheadQueue.add(buffer); |
|
||||
notifyAll(); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}", |
|
||||
stream.getPath(), requestedOffset, buffer.getBufferindex()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a |
|
||||
* remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading |
|
||||
* the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead |
|
||||
* but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because |
|
||||
* depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own |
|
||||
* read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time). |
|
||||
* |
|
||||
* @param stream the file to read bytes for |
|
||||
* @param position the offset in the file to do a read for |
|
||||
* @param length the length to read |
|
||||
* @param buffer the buffer to read data into. Note that the buffer will be written into from offset 0. |
|
||||
* @return the number of bytes read |
|
||||
*/ |
|
||||
int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) { |
|
||||
// not synchronized, so have to be careful with locking |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("getBlock for file {} position {} thread {}", |
|
||||
stream.getPath(), position, Thread.currentThread().getName()); |
|
||||
} |
|
||||
|
|
||||
waitForProcess(stream, position); |
|
||||
|
|
||||
int bytesRead = 0; |
|
||||
synchronized (this) { |
|
||||
bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer); |
|
||||
} |
|
||||
if (bytesRead > 0) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Done read from Cache for {} position {} length {}", |
|
||||
stream.getPath(), position, bytesRead); |
|
||||
} |
|
||||
return bytesRead; |
|
||||
} |
|
||||
|
|
||||
// otherwise, just say we got nothing - calling thread can do its own read |
|
||||
return 0; |
|
||||
} |
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* Internal methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
private void waitForProcess(final SeaweedInputStream stream, final long position) { |
|
||||
ReadBuffer readBuf; |
|
||||
synchronized (this) { |
|
||||
clearFromReadAheadQueue(stream, position); |
|
||||
readBuf = getFromList(inProgressList, stream, position); |
|
||||
} |
|
||||
if (readBuf != null) { // if in in-progress queue, then block for it |
|
||||
try { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}", |
|
||||
stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex()); |
|
||||
} |
|
||||
readBuf.getLatch().await(); // blocking wait on the caller stream's thread |
|
||||
// Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread |
|
||||
// is done processing it (in doneReading). There, the latch is set after removing the buffer from |
|
||||
// inProgressList. So this latch is safe to be outside the synchronized block. |
|
||||
// Putting it in synchronized would result in a deadlock, since this thread would be holding the lock |
|
||||
// while waiting, so no one will be able to change any state. If this becomes more complex in the future, |
|
||||
// then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched. |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("latch done for file {} buffer idx {} length {}", |
|
||||
stream.getPath(), readBuf.getBufferindex(), readBuf.getLength()); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list. |
|
||||
* The objective is to find just one buffer - there is no advantage to evicting more than one. |
|
||||
* |
|
||||
* @return whether the eviction succeeeded - i.e., were we able to free up one buffer |
|
||||
*/ |
|
||||
private synchronized boolean tryEvict() { |
|
||||
ReadBuffer nodeToEvict = null; |
|
||||
if (completedReadList.size() <= 0) { |
|
||||
return false; // there are no evict-able buffers |
|
||||
} |
|
||||
|
|
||||
// first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed) |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) { |
|
||||
nodeToEvict = buf; |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
if (nodeToEvict != null) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see) |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.isAnyByteConsumed()) { |
|
||||
nodeToEvict = buf; |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
if (nodeToEvict != null) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// next, try any old nodes that have not been consumed |
|
||||
long earliestBirthday = Long.MAX_VALUE; |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.getTimeStamp() < earliestBirthday) { |
|
||||
nodeToEvict = buf; |
|
||||
earliestBirthday = buf.getTimeStamp(); |
|
||||
} |
|
||||
} |
|
||||
if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// nothing can be evicted |
|
||||
return false; |
|
||||
} |
|
||||
|
|
||||
private boolean evict(final ReadBuffer buf) { |
|
||||
freeList.push(buf.getBufferindex()); |
|
||||
completedReadList.remove(buf); |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}", |
|
||||
buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength()); |
|
||||
} |
|
||||
return true; |
|
||||
} |
|
||||
|
|
||||
private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
// returns true if any part of the buffer is already queued |
|
||||
return (isInList(readAheadQueue, stream, requestedOffset) |
|
||||
|| isInList(inProgressList, stream, requestedOffset) |
|
||||
|| isInList(completedReadList, stream, requestedOffset)); |
|
||||
} |
|
||||
|
|
||||
private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
return (getFromList(list, stream, requestedOffset) != null); |
|
||||
} |
|
||||
|
|
||||
private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
for (ReadBuffer buffer : list) { |
|
||||
if (buffer.getStream() == stream) { |
|
||||
if (buffer.getStatus() == ReadBufferStatus.AVAILABLE |
|
||||
&& requestedOffset >= buffer.getOffset() |
|
||||
&& requestedOffset < buffer.getOffset() + buffer.getLength()) { |
|
||||
return buffer; |
|
||||
} else if (requestedOffset >= buffer.getOffset() |
|
||||
&& requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) { |
|
||||
return buffer; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
return null; |
|
||||
} |
|
||||
|
|
||||
private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset); |
|
||||
if (buffer != null) { |
|
||||
readAheadQueue.remove(buffer); |
|
||||
notifyAll(); // lock is held in calling method |
|
||||
freeList.push(buffer.getBufferindex()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length, |
|
||||
final byte[] buffer) { |
|
||||
ReadBuffer buf = getFromList(completedReadList, stream, position); |
|
||||
if (buf == null || position >= buf.getOffset() + buf.getLength()) { |
|
||||
return 0; |
|
||||
} |
|
||||
int cursor = (int) (position - buf.getOffset()); |
|
||||
int availableLengthInBuffer = buf.getLength() - cursor; |
|
||||
int lengthToCopy = Math.min(length, availableLengthInBuffer); |
|
||||
System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy); |
|
||||
if (cursor == 0) { |
|
||||
buf.setFirstByteConsumed(true); |
|
||||
} |
|
||||
if (cursor + lengthToCopy == buf.getLength()) { |
|
||||
buf.setLastByteConsumed(true); |
|
||||
} |
|
||||
buf.setAnyByteConsumed(true); |
|
||||
return lengthToCopy; |
|
||||
} |
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* ReadBufferWorker-thread-facing methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
/** |
|
||||
* ReadBufferWorker thread calls this to get the next buffer that it should work on. |
|
||||
* |
|
||||
* @return {@link ReadBuffer} |
|
||||
* @throws InterruptedException if thread is interrupted |
|
||||
*/ |
|
||||
ReadBuffer getNextBlockToRead() throws InterruptedException { |
|
||||
ReadBuffer buffer = null; |
|
||||
synchronized (this) { |
|
||||
//buffer = readAheadQueue.take(); // blocking method |
|
||||
while (readAheadQueue.size() == 0) { |
|
||||
wait(); |
|
||||
} |
|
||||
buffer = readAheadQueue.remove(); |
|
||||
notifyAll(); |
|
||||
if (buffer == null) { |
|
||||
return null; // should never happen |
|
||||
} |
|
||||
buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS); |
|
||||
inProgressList.add(buffer); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("ReadBufferWorker picked file {} for offset {}", |
|
||||
buffer.getStream().getPath(), buffer.getOffset()); |
|
||||
} |
|
||||
return buffer; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* ReadBufferWorker thread calls this method to post completion. |
|
||||
* |
|
||||
* @param buffer the buffer whose read was completed |
|
||||
* @param result the {@link ReadBufferStatus} after the read operation in the worker thread |
|
||||
* @param bytesActuallyRead the number of bytes that the worker thread was actually able to read |
|
||||
*/ |
|
||||
void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}", |
|
||||
buffer.getStream().getPath(), buffer.getOffset(), bytesActuallyRead); |
|
||||
} |
|
||||
synchronized (this) { |
|
||||
inProgressList.remove(buffer); |
|
||||
if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { |
|
||||
buffer.setStatus(ReadBufferStatus.AVAILABLE); |
|
||||
buffer.setTimeStamp(currentTimeMillis()); |
|
||||
buffer.setLength(bytesActuallyRead); |
|
||||
completedReadList.add(buffer); |
|
||||
} else { |
|
||||
freeList.push(buffer.getBufferindex()); |
|
||||
// buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC |
|
||||
} |
|
||||
} |
|
||||
//outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results |
|
||||
buffer.getLatch().countDown(); // wake up waiting threads (if any) |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Similar to System.currentTimeMillis, except implemented with System.nanoTime(). |
|
||||
* System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization), |
|
||||
* making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core. |
|
||||
* Note: it is not monotonic across Sockets, and even within a CPU, its only the |
|
||||
* more recent parts which share a clock across all cores. |
|
||||
* |
|
||||
* @return current time in milliseconds |
|
||||
*/ |
|
||||
private long currentTimeMillis() { |
|
||||
return System.nanoTime() / 1000 / 1000; |
|
||||
} |
|
||||
} |
|
||||
@ -1,70 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
|
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
class ReadBufferWorker implements Runnable { |
|
||||
|
|
||||
protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1); |
|
||||
private int id; |
|
||||
|
|
||||
ReadBufferWorker(final int id) { |
|
||||
this.id = id; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* return the ID of ReadBufferWorker. |
|
||||
*/ |
|
||||
public int getId() { |
|
||||
return this.id; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Waits until a buffer becomes available in ReadAheadQueue. |
|
||||
* Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager. |
|
||||
* Rinse and repeat. Forever. |
|
||||
*/ |
|
||||
public void run() { |
|
||||
try { |
|
||||
UNLEASH_WORKERS.await(); |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
} |
|
||||
ReadBufferManager bufferManager = ReadBufferManager.getBufferManager(); |
|
||||
ReadBuffer buffer; |
|
||||
while (true) { |
|
||||
try { |
|
||||
buffer = bufferManager.getNextBlockToRead(); // blocks, until a buffer is available for this thread |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
return; |
|
||||
} |
|
||||
if (buffer != null) { |
|
||||
try { |
|
||||
// do the actual read, from the file. |
|
||||
int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength()); |
|
||||
bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead); // post result back to ReadBufferManager |
|
||||
} catch (Exception ex) { |
|
||||
bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -1,137 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
|
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
class ReadBuffer { |
|
||||
|
|
||||
private SeaweedInputStream stream; |
|
||||
private long offset; // offset within the file for the buffer |
|
||||
private int length; // actual length, set after the buffer is filles |
|
||||
private int requestedLength; // requested length of the read |
|
||||
private byte[] buffer; // the buffer itself |
|
||||
private int bufferindex = -1; // index in the buffers array in Buffer manager |
|
||||
private ReadBufferStatus status; // status of the buffer |
|
||||
private CountDownLatch latch = null; // signaled when the buffer is done reading, so any client |
|
||||
// waiting on this buffer gets unblocked |
|
||||
|
|
||||
// fields to help with eviction logic |
|
||||
private long timeStamp = 0; // tick at which buffer became available to read |
|
||||
private boolean isFirstByteConsumed = false; |
|
||||
private boolean isLastByteConsumed = false; |
|
||||
private boolean isAnyByteConsumed = false; |
|
||||
|
|
||||
public SeaweedInputStream getStream() { |
|
||||
return stream; |
|
||||
} |
|
||||
|
|
||||
public void setStream(SeaweedInputStream stream) { |
|
||||
this.stream = stream; |
|
||||
} |
|
||||
|
|
||||
public long getOffset() { |
|
||||
return offset; |
|
||||
} |
|
||||
|
|
||||
public void setOffset(long offset) { |
|
||||
this.offset = offset; |
|
||||
} |
|
||||
|
|
||||
public int getLength() { |
|
||||
return length; |
|
||||
} |
|
||||
|
|
||||
public void setLength(int length) { |
|
||||
this.length = length; |
|
||||
} |
|
||||
|
|
||||
public int getRequestedLength() { |
|
||||
return requestedLength; |
|
||||
} |
|
||||
|
|
||||
public void setRequestedLength(int requestedLength) { |
|
||||
this.requestedLength = requestedLength; |
|
||||
} |
|
||||
|
|
||||
public byte[] getBuffer() { |
|
||||
return buffer; |
|
||||
} |
|
||||
|
|
||||
public void setBuffer(byte[] buffer) { |
|
||||
this.buffer = buffer; |
|
||||
} |
|
||||
|
|
||||
public int getBufferindex() { |
|
||||
return bufferindex; |
|
||||
} |
|
||||
|
|
||||
public void setBufferindex(int bufferindex) { |
|
||||
this.bufferindex = bufferindex; |
|
||||
} |
|
||||
|
|
||||
public ReadBufferStatus getStatus() { |
|
||||
return status; |
|
||||
} |
|
||||
|
|
||||
public void setStatus(ReadBufferStatus status) { |
|
||||
this.status = status; |
|
||||
} |
|
||||
|
|
||||
public CountDownLatch getLatch() { |
|
||||
return latch; |
|
||||
} |
|
||||
|
|
||||
public void setLatch(CountDownLatch latch) { |
|
||||
this.latch = latch; |
|
||||
} |
|
||||
|
|
||||
public long getTimeStamp() { |
|
||||
return timeStamp; |
|
||||
} |
|
||||
|
|
||||
public void setTimeStamp(long timeStamp) { |
|
||||
this.timeStamp = timeStamp; |
|
||||
} |
|
||||
|
|
||||
public boolean isFirstByteConsumed() { |
|
||||
return isFirstByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setFirstByteConsumed(boolean isFirstByteConsumed) { |
|
||||
this.isFirstByteConsumed = isFirstByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public boolean isLastByteConsumed() { |
|
||||
return isLastByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setLastByteConsumed(boolean isLastByteConsumed) { |
|
||||
this.isLastByteConsumed = isLastByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public boolean isAnyByteConsumed() { |
|
||||
return isAnyByteConsumed; |
|
||||
} |
|
||||
|
|
||||
public void setAnyByteConsumed(boolean isAnyByteConsumed) { |
|
||||
this.isAnyByteConsumed = isAnyByteConsumed; |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,394 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* <p> |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* <p> |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.Collection; |
|
||||
import java.util.LinkedList; |
|
||||
import java.util.Queue; |
|
||||
import java.util.Stack; |
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
/** |
|
||||
* The Read Buffer Manager for Rest AbfsClient. |
|
||||
*/ |
|
||||
final class ReadBufferManager { |
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class); |
|
||||
|
|
||||
private static final int NUM_BUFFERS = 16; |
|
||||
private static final int BLOCK_SIZE = 4 * 1024 * 1024; |
|
||||
private static final int NUM_THREADS = 8; |
|
||||
private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold |
|
||||
|
|
||||
private Thread[] threads = new Thread[NUM_THREADS]; |
|
||||
private byte[][] buffers; // array of byte[] buffers, to hold the data that is read |
|
||||
private Stack<Integer> freeList = new Stack<>(); // indices in buffers[] array that are available |
|
||||
|
|
||||
private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet |
|
||||
private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads |
|
||||
private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading |
|
||||
private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block |
|
||||
|
|
||||
static { |
|
||||
BUFFER_MANAGER = new ReadBufferManager(); |
|
||||
BUFFER_MANAGER.init(); |
|
||||
} |
|
||||
|
|
||||
static ReadBufferManager getBufferManager() { |
|
||||
return BUFFER_MANAGER; |
|
||||
} |
|
||||
|
|
||||
private void init() { |
|
||||
buffers = new byte[NUM_BUFFERS][]; |
|
||||
for (int i = 0; i < NUM_BUFFERS; i++) { |
|
||||
buffers[i] = new byte[BLOCK_SIZE]; // same buffers are reused. The byte array never goes back to GC |
|
||||
freeList.add(i); |
|
||||
} |
|
||||
for (int i = 0; i < NUM_THREADS; i++) { |
|
||||
Thread t = new Thread(new ReadBufferWorker(i)); |
|
||||
t.setDaemon(true); |
|
||||
threads[i] = t; |
|
||||
t.setName("SeaweedFS-prefetch-" + i); |
|
||||
t.start(); |
|
||||
} |
|
||||
ReadBufferWorker.UNLEASH_WORKERS.countDown(); |
|
||||
} |
|
||||
|
|
||||
// hide instance constructor |
|
||||
private ReadBufferManager() { |
|
||||
} |
|
||||
|
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* SeaweedInputStream-facing methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* {@link SeaweedInputStream} calls this method to queue read-aheads. |
|
||||
* |
|
||||
* @param stream The {@link SeaweedInputStream} for which to do the read-ahead |
|
||||
* @param requestedOffset The offset in the file which shoukd be read |
|
||||
* @param requestedLength The length to read |
|
||||
*/ |
|
||||
void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Start Queueing readAhead for {} offset {} length {}", |
|
||||
stream.getPath(), requestedOffset, requestedLength); |
|
||||
} |
|
||||
ReadBuffer buffer; |
|
||||
synchronized (this) { |
|
||||
if (isAlreadyQueued(stream, requestedOffset)) { |
|
||||
return; // already queued, do not queue again |
|
||||
} |
|
||||
if (freeList.isEmpty() && !tryEvict()) { |
|
||||
return; // no buffers available, cannot queue anything |
|
||||
} |
|
||||
|
|
||||
buffer = new ReadBuffer(); |
|
||||
buffer.setStream(stream); |
|
||||
buffer.setOffset(requestedOffset); |
|
||||
buffer.setLength(0); |
|
||||
buffer.setRequestedLength(requestedLength); |
|
||||
buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE); |
|
||||
buffer.setLatch(new CountDownLatch(1)); |
|
||||
|
|
||||
Integer bufferIndex = freeList.pop(); // will return a value, since we have checked size > 0 already |
|
||||
|
|
||||
buffer.setBuffer(buffers[bufferIndex]); |
|
||||
buffer.setBufferindex(bufferIndex); |
|
||||
readAheadQueue.add(buffer); |
|
||||
notifyAll(); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}", |
|
||||
stream.getPath(), requestedOffset, buffer.getBufferindex()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a |
|
||||
* remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading |
|
||||
* the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead |
|
||||
* but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because |
|
||||
* depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own |
|
||||
* read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time). |
|
||||
* |
|
||||
* @param stream the file to read bytes for |
|
||||
* @param position the offset in the file to do a read for |
|
||||
* @param length the length to read |
|
||||
* @param buffer the buffer to read data into. Note that the buffer will be written into from offset 0. |
|
||||
* @return the number of bytes read |
|
||||
*/ |
|
||||
int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) { |
|
||||
// not synchronized, so have to be careful with locking |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("getBlock for file {} position {} thread {}", |
|
||||
stream.getPath(), position, Thread.currentThread().getName()); |
|
||||
} |
|
||||
|
|
||||
waitForProcess(stream, position); |
|
||||
|
|
||||
int bytesRead = 0; |
|
||||
synchronized (this) { |
|
||||
bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer); |
|
||||
} |
|
||||
if (bytesRead > 0) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Done read from Cache for {} position {} length {}", |
|
||||
stream.getPath(), position, bytesRead); |
|
||||
} |
|
||||
return bytesRead; |
|
||||
} |
|
||||
|
|
||||
// otherwise, just say we got nothing - calling thread can do its own read |
|
||||
return 0; |
|
||||
} |
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* Internal methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
private void waitForProcess(final SeaweedInputStream stream, final long position) { |
|
||||
ReadBuffer readBuf; |
|
||||
synchronized (this) { |
|
||||
clearFromReadAheadQueue(stream, position); |
|
||||
readBuf = getFromList(inProgressList, stream, position); |
|
||||
} |
|
||||
if (readBuf != null) { // if in in-progress queue, then block for it |
|
||||
try { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}", |
|
||||
stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex()); |
|
||||
} |
|
||||
readBuf.getLatch().await(); // blocking wait on the caller stream's thread |
|
||||
// Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread |
|
||||
// is done processing it (in doneReading). There, the latch is set after removing the buffer from |
|
||||
// inProgressList. So this latch is safe to be outside the synchronized block. |
|
||||
// Putting it in synchronized would result in a deadlock, since this thread would be holding the lock |
|
||||
// while waiting, so no one will be able to change any state. If this becomes more complex in the future, |
|
||||
// then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched. |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("latch done for file {} buffer idx {} length {}", |
|
||||
stream.getPath(), readBuf.getBufferindex(), readBuf.getLength()); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list. |
|
||||
* The objective is to find just one buffer - there is no advantage to evicting more than one. |
|
||||
* |
|
||||
* @return whether the eviction succeeeded - i.e., were we able to free up one buffer |
|
||||
*/ |
|
||||
private synchronized boolean tryEvict() { |
|
||||
ReadBuffer nodeToEvict = null; |
|
||||
if (completedReadList.size() <= 0) { |
|
||||
return false; // there are no evict-able buffers |
|
||||
} |
|
||||
|
|
||||
// first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed) |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) { |
|
||||
nodeToEvict = buf; |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
if (nodeToEvict != null) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see) |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.isAnyByteConsumed()) { |
|
||||
nodeToEvict = buf; |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
if (nodeToEvict != null) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// next, try any old nodes that have not been consumed |
|
||||
long earliestBirthday = Long.MAX_VALUE; |
|
||||
for (ReadBuffer buf : completedReadList) { |
|
||||
if (buf.getTimeStamp() < earliestBirthday) { |
|
||||
nodeToEvict = buf; |
|
||||
earliestBirthday = buf.getTimeStamp(); |
|
||||
} |
|
||||
} |
|
||||
if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) { |
|
||||
return evict(nodeToEvict); |
|
||||
} |
|
||||
|
|
||||
// nothing can be evicted |
|
||||
return false; |
|
||||
} |
|
||||
|
|
||||
private boolean evict(final ReadBuffer buf) { |
|
||||
freeList.push(buf.getBufferindex()); |
|
||||
completedReadList.remove(buf); |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}", |
|
||||
buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength()); |
|
||||
} |
|
||||
return true; |
|
||||
} |
|
||||
|
|
||||
private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
// returns true if any part of the buffer is already queued |
|
||||
return (isInList(readAheadQueue, stream, requestedOffset) |
|
||||
|| isInList(inProgressList, stream, requestedOffset) |
|
||||
|| isInList(completedReadList, stream, requestedOffset)); |
|
||||
} |
|
||||
|
|
||||
private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
return (getFromList(list, stream, requestedOffset) != null); |
|
||||
} |
|
||||
|
|
||||
private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
for (ReadBuffer buffer : list) { |
|
||||
if (buffer.getStream() == stream) { |
|
||||
if (buffer.getStatus() == ReadBufferStatus.AVAILABLE |
|
||||
&& requestedOffset >= buffer.getOffset() |
|
||||
&& requestedOffset < buffer.getOffset() + buffer.getLength()) { |
|
||||
return buffer; |
|
||||
} else if (requestedOffset >= buffer.getOffset() |
|
||||
&& requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) { |
|
||||
return buffer; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
return null; |
|
||||
} |
|
||||
|
|
||||
private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) { |
|
||||
ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset); |
|
||||
if (buffer != null) { |
|
||||
readAheadQueue.remove(buffer); |
|
||||
notifyAll(); // lock is held in calling method |
|
||||
freeList.push(buffer.getBufferindex()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length, |
|
||||
final byte[] buffer) { |
|
||||
ReadBuffer buf = getFromList(completedReadList, stream, position); |
|
||||
if (buf == null || position >= buf.getOffset() + buf.getLength()) { |
|
||||
return 0; |
|
||||
} |
|
||||
int cursor = (int) (position - buf.getOffset()); |
|
||||
int availableLengthInBuffer = buf.getLength() - cursor; |
|
||||
int lengthToCopy = Math.min(length, availableLengthInBuffer); |
|
||||
System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy); |
|
||||
if (cursor == 0) { |
|
||||
buf.setFirstByteConsumed(true); |
|
||||
} |
|
||||
if (cursor + lengthToCopy == buf.getLength()) { |
|
||||
buf.setLastByteConsumed(true); |
|
||||
} |
|
||||
buf.setAnyByteConsumed(true); |
|
||||
return lengthToCopy; |
|
||||
} |
|
||||
|
|
||||
/* |
|
||||
* |
|
||||
* ReadBufferWorker-thread-facing methods |
|
||||
* |
|
||||
*/ |
|
||||
|
|
||||
/** |
|
||||
* ReadBufferWorker thread calls this to get the next buffer that it should work on. |
|
||||
* |
|
||||
* @return {@link ReadBuffer} |
|
||||
* @throws InterruptedException if thread is interrupted |
|
||||
*/ |
|
||||
ReadBuffer getNextBlockToRead() throws InterruptedException { |
|
||||
ReadBuffer buffer = null; |
|
||||
synchronized (this) { |
|
||||
//buffer = readAheadQueue.take(); // blocking method |
|
||||
while (readAheadQueue.size() == 0) { |
|
||||
wait(); |
|
||||
} |
|
||||
buffer = readAheadQueue.remove(); |
|
||||
notifyAll(); |
|
||||
if (buffer == null) { |
|
||||
return null; // should never happen |
|
||||
} |
|
||||
buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS); |
|
||||
inProgressList.add(buffer); |
|
||||
} |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("ReadBufferWorker picked file {} for offset {}", |
|
||||
buffer.getStream().getPath(), buffer.getOffset()); |
|
||||
} |
|
||||
return buffer; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* ReadBufferWorker thread calls this method to post completion. |
|
||||
* |
|
||||
* @param buffer the buffer whose read was completed |
|
||||
* @param result the {@link ReadBufferStatus} after the read operation in the worker thread |
|
||||
* @param bytesActuallyRead the number of bytes that the worker thread was actually able to read |
|
||||
*/ |
|
||||
void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) { |
|
||||
if (LOGGER.isTraceEnabled()) { |
|
||||
LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}", |
|
||||
buffer.getStream().getPath(), buffer.getOffset(), bytesActuallyRead); |
|
||||
} |
|
||||
synchronized (this) { |
|
||||
inProgressList.remove(buffer); |
|
||||
if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) { |
|
||||
buffer.setStatus(ReadBufferStatus.AVAILABLE); |
|
||||
buffer.setTimeStamp(currentTimeMillis()); |
|
||||
buffer.setLength(bytesActuallyRead); |
|
||||
completedReadList.add(buffer); |
|
||||
} else { |
|
||||
freeList.push(buffer.getBufferindex()); |
|
||||
// buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC |
|
||||
} |
|
||||
} |
|
||||
//outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results |
|
||||
buffer.getLatch().countDown(); // wake up waiting threads (if any) |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Similar to System.currentTimeMillis, except implemented with System.nanoTime(). |
|
||||
* System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization), |
|
||||
* making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core. |
|
||||
* Note: it is not monotonic across Sockets, and even within a CPU, its only the |
|
||||
* more recent parts which share a clock across all cores. |
|
||||
* |
|
||||
* @return current time in milliseconds |
|
||||
*/ |
|
||||
private long currentTimeMillis() { |
|
||||
return System.nanoTime() / 1000 / 1000; |
|
||||
} |
|
||||
} |
|
||||
@ -1,70 +0,0 @@ |
|||||
/** |
|
||||
* Licensed to the Apache Software Foundation (ASF) under one |
|
||||
* or more contributor license agreements. See the NOTICE file |
|
||||
* distributed with this work for additional information |
|
||||
* regarding copyright ownership. The ASF licenses this file |
|
||||
* to you under the Apache License, Version 2.0 (the |
|
||||
* "License"); you may not use this file except in compliance |
|
||||
* with the License. You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
|
|
||||
package seaweed.hdfs; |
|
||||
|
|
||||
import java.util.concurrent.CountDownLatch; |
|
||||
|
|
||||
class ReadBufferWorker implements Runnable { |
|
||||
|
|
||||
protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1); |
|
||||
private int id; |
|
||||
|
|
||||
ReadBufferWorker(final int id) { |
|
||||
this.id = id; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* return the ID of ReadBufferWorker. |
|
||||
*/ |
|
||||
public int getId() { |
|
||||
return this.id; |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* Waits until a buffer becomes available in ReadAheadQueue. |
|
||||
* Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager. |
|
||||
* Rinse and repeat. Forever. |
|
||||
*/ |
|
||||
public void run() { |
|
||||
try { |
|
||||
UNLEASH_WORKERS.await(); |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
} |
|
||||
ReadBufferManager bufferManager = ReadBufferManager.getBufferManager(); |
|
||||
ReadBuffer buffer; |
|
||||
while (true) { |
|
||||
try { |
|
||||
buffer = bufferManager.getNextBlockToRead(); // blocks, until a buffer is available for this thread |
|
||||
} catch (InterruptedException ex) { |
|
||||
Thread.currentThread().interrupt(); |
|
||||
return; |
|
||||
} |
|
||||
if (buffer != null) { |
|
||||
try { |
|
||||
// do the actual read, from the file. |
|
||||
int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength()); |
|
||||
bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead); // post result back to ReadBufferManager |
|
||||
} catch (Exception ex) { |
|
||||
bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,111 @@ |
|||||
|
package basic |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"os" |
||||
|
"strings" |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/aws/aws-sdk-go/aws" |
||||
|
"github.com/aws/aws-sdk-go/aws/awserr" |
||||
|
"github.com/aws/aws-sdk-go/aws/session" |
||||
|
"github.com/aws/aws-sdk-go/service/s3" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
svc *s3.S3 |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
// Initialize a session in us-west-2 that the SDK will use to load
|
||||
|
// credentials from the shared credentials file ~/.aws/credentials.
|
||||
|
sess, err := session.NewSession(&aws.Config{ |
||||
|
Region: aws.String("us-west-2"), |
||||
|
Endpoint: aws.String("localhost:8333"), |
||||
|
DisableSSL: aws.Bool(true), |
||||
|
}) |
||||
|
if err != nil { |
||||
|
exitErrorf("create session, %v", err) |
||||
|
} |
||||
|
|
||||
|
// Create S3 service client
|
||||
|
svc = s3.New(sess) |
||||
|
} |
||||
|
|
||||
|
func TestCreateBucket(t *testing.T) { |
||||
|
|
||||
|
input := &s3.CreateBucketInput{ |
||||
|
Bucket: aws.String("theBucket"), |
||||
|
} |
||||
|
|
||||
|
result, err := svc.CreateBucket(input) |
||||
|
if err != nil { |
||||
|
if aerr, ok := err.(awserr.Error); ok { |
||||
|
switch aerr.Code() { |
||||
|
case s3.ErrCodeBucketAlreadyExists: |
||||
|
fmt.Println(s3.ErrCodeBucketAlreadyExists, aerr.Error()) |
||||
|
case s3.ErrCodeBucketAlreadyOwnedByYou: |
||||
|
fmt.Println(s3.ErrCodeBucketAlreadyOwnedByYou, aerr.Error()) |
||||
|
default: |
||||
|
fmt.Println(aerr.Error()) |
||||
|
} |
||||
|
} else { |
||||
|
// Print the error, cast err to awserr.Error to get the Code and
|
||||
|
// Message from an error.
|
||||
|
fmt.Println(err.Error()) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
fmt.Println(result) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func TestListBuckets(t *testing.T) { |
||||
|
|
||||
|
input := &s3.PutObjectInput{ |
||||
|
ACL: aws.String("authenticated-read"), |
||||
|
Body: aws.ReadSeekCloser(strings.NewReader("filetoupload")), |
||||
|
Bucket: aws.String("theBucket"), |
||||
|
Key: aws.String("exampleobject"), |
||||
|
} |
||||
|
|
||||
|
result, err := svc.PutObject(input) |
||||
|
if err != nil { |
||||
|
if aerr, ok := err.(awserr.Error); ok { |
||||
|
switch aerr.Code() { |
||||
|
default: |
||||
|
fmt.Println(aerr.Error()) |
||||
|
} |
||||
|
} else { |
||||
|
// Print the error, cast err to awserr.Error to get the Code and
|
||||
|
// Message from an error.
|
||||
|
fmt.Println(err.Error()) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
fmt.Println(result) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func TestPutObject(t *testing.T) { |
||||
|
|
||||
|
result, err := svc.ListBuckets(nil) |
||||
|
if err != nil { |
||||
|
exitErrorf("Unable to list buckets, %v", err) |
||||
|
} |
||||
|
|
||||
|
fmt.Println("Buckets:") |
||||
|
|
||||
|
for _, b := range result.Buckets { |
||||
|
fmt.Printf("* %s created on %s\n", |
||||
|
aws.StringValue(b.Name), aws.TimeValue(b.CreationDate)) |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func exitErrorf(msg string, args ...interface{}) { |
||||
|
fmt.Fprintf(os.Stderr, msg+"\n", args...) |
||||
|
os.Exit(1) |
||||
|
} |
||||
@ -0,0 +1,43 @@ |
|||||
|
package leveldb |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/filer2" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
_ = filer2.FilerLocalStore(&LevelDB2Store{}) |
||||
|
) |
||||
|
|
||||
|
func (store *LevelDB2Store) UpdateOffset(filer string, lastTsNs int64) error { |
||||
|
|
||||
|
value := make([]byte, 8) |
||||
|
util.Uint64toBytes(value, uint64(lastTsNs)) |
||||
|
|
||||
|
err := store.dbs[0].Put([]byte("meta"+filer), value, nil) |
||||
|
|
||||
|
if err != nil { |
||||
|
return fmt.Errorf("UpdateOffset %s : %v", filer, err) |
||||
|
} |
||||
|
|
||||
|
println("UpdateOffset", filer, "lastTsNs", lastTsNs) |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDB2Store) ReadOffset(filer string) (lastTsNs int64, err error) { |
||||
|
|
||||
|
value, err := store.dbs[0].Get([]byte("meta"+filer), nil) |
||||
|
|
||||
|
if err != nil { |
||||
|
return 0, fmt.Errorf("ReadOffset %s : %v", filer, err) |
||||
|
} |
||||
|
|
||||
|
lastTsNs = int64(util.BytesToUint64(value)) |
||||
|
|
||||
|
println("ReadOffset", filer, "lastTsNs", lastTsNs) |
||||
|
|
||||
|
return |
||||
|
} |
||||
@ -0,0 +1,37 @@ |
|||||
|
package filer2 |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) error { |
||||
|
message := resp.EventNotification |
||||
|
var oldPath util.FullPath |
||||
|
var newEntry *Entry |
||||
|
if message.OldEntry != nil { |
||||
|
oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name) |
||||
|
glog.V(4).Infof("deleting %v", oldPath) |
||||
|
if err := filerStore.DeleteEntry(context.Background(), oldPath); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if message.NewEntry != nil { |
||||
|
dir := resp.Directory |
||||
|
if message.NewParentPath != "" { |
||||
|
dir = message.NewParentPath |
||||
|
} |
||||
|
key := util.NewFullPath(dir, message.NewEntry.Name) |
||||
|
glog.V(4).Infof("creating %v", key) |
||||
|
newEntry = FromPbEntry(dir, message.NewEntry) |
||||
|
if err := filerStore.InsertEntry(context.Background(), newEntry); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
1162
weed/pb/filer_pb/filer.pb.go
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Write
Preview
Loading…
Cancel
Save
Reference in new issue