mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
hilimd
GitHub
commit
218561c206
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
|
|
@ -15,7 +15,7 @@ jobs:
|
|||
uses: actions/checkout@v2
|
||||
|
||||
- name: Go Release Binaries
|
||||
uses: wangyoucao577/go-release-action@feature/customize-target-release
|
||||
uses: wangyoucao577/go-release-action@v1.10
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: linux # default is
|
||||
|
|
@ -28,7 +28,7 @@ jobs:
|
|||
project_path: weed
|
||||
binary_name: weed-large-disk
|
||||
- name: Go Release Binaries
|
||||
uses: wangyoucao577/go-release-action@feature/customize-target-release
|
||||
uses: wangyoucao577/go-release-action@v1.10
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: linux # default is
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -11,7 +11,7 @@ require (
|
|||
github.com/aws/aws-sdk-go v1.33.5
|
||||
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011
|
||||
github.com/chrislusf/raft v1.0.3
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
|
|
@ -24,6 +24,7 @@ require (
|
|||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.8.0
|
||||
github.com/frankban/quicktest v1.7.2 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis v6.15.7+incompatible
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
|
||||
|
|
@ -45,6 +46,7 @@ require (
|
|||
github.com/klauspost/reedsolomon v1.9.2
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/lunixbochs/vtclean v1.0.0 // indirect
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
|
|
@ -67,6 +69,9 @@ require (
|
|||
github.com/tidwall/gjson v1.3.2
|
||||
github.com/tidwall/match v1.0.1
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/assertly v0.5.4 // indirect
|
||||
github.com/viant/ptrie v0.3.0
|
||||
github.com/viant/toolbox v0.33.2 // indirect
|
||||
github.com/willf/bitset v1.1.10 // indirect
|
||||
github.com/willf/bloom v2.0.3+incompatible
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -84,8 +84,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
|||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIPCdhuVKAjlYpxG1B86jiKejB6MC/Q0=
|
||||
github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
|
||||
github.com/chrislusf/raft v1.0.3 h1:11YrnzJtVa5z7m9lhY2p8VcPHoUlC1UswyoAo+U1m1k=
|
||||
github.com/chrislusf/raft v1.0.3/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68=
|
||||
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
|
|
@ -159,6 +159,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
|
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
|
||||
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
|
||||
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
|
|
@ -386,6 +388,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
|
|||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
|
|
@ -621,6 +625,12 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
|
|||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/viant/assertly v0.5.4 h1:5Hh4U3pLZa6uhCFAGpYOxck/8l9TZczEzoHNfJAhHEQ=
|
||||
github.com/viant/assertly v0.5.4/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/ptrie v0.3.0 h1:SDaRd7Gqr1+ItCNz0GpTxRdK21nOfqjV6YtBm9jGlMY=
|
||||
github.com/viant/ptrie v0.3.0/go.mod h1:VguMnbGfz95Zw+V5VarYSqtqslDxJbOv++xLzxkMhec=
|
||||
github.com/viant/toolbox v0.33.2 h1:Av844IIeGz81gT672qZemyptGfbrcxqGymA5RFnIPjE=
|
||||
github.com/viant/toolbox v0.33.2/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
|
||||
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
version: 2.10
|
||||
version: 2.11
|
||||
|
|
@ -4,7 +4,7 @@ global:
|
|||
registry: ""
|
||||
repository: ""
|
||||
imageName: chrislusf/seaweedfs
|
||||
imageTag: "2.10"
|
||||
imageTag: "2.11"
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: imagepullsecret
|
||||
restartPolicy: Always
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
<version>1.5.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
<version>1.5.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
<version>1.5.4</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
|
|
|
|||
|
|
@ -275,9 +275,9 @@ public class FilerClient {
|
|||
try {
|
||||
FilerProto.CreateEntryResponse createEntryResponse =
|
||||
filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
|
||||
.setDirectory(parent)
|
||||
.setEntry(entry)
|
||||
.build());
|
||||
.setDirectory(parent)
|
||||
.setEntry(entry)
|
||||
.build());
|
||||
if (Strings.isNullOrEmpty(createEntryResponse.getError())) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -333,4 +333,13 @@ public class FilerClient {
|
|||
return true;
|
||||
}
|
||||
|
||||
public Iterator<FilerProto.SubscribeMetadataResponse> watch(String prefix, String clientName, long sinceNs) {
|
||||
return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder()
|
||||
.setPathPrefix(prefix)
|
||||
.setClientName(clientName)
|
||||
.setSinceNs(sinceNs)
|
||||
.build()
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,20 +28,26 @@ public class SeaweedRead {
|
|||
|
||||
List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength);
|
||||
|
||||
Map<String, FilerProto.Locations> knownLocations = new HashMap<>();
|
||||
|
||||
FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
|
||||
for (ChunkView chunkView : chunkViews) {
|
||||
String vid = parseVolumeId(chunkView.fileId);
|
||||
if (volumeIdCache.getLocations(vid)==null){
|
||||
FilerProto.Locations locations = volumeIdCache.getLocations(vid);
|
||||
if (locations == null) {
|
||||
lookupRequest.addVolumeIds(vid);
|
||||
} else {
|
||||
knownLocations.put(vid, locations);
|
||||
}
|
||||
}
|
||||
|
||||
if (lookupRequest.getVolumeIdsCount()>0){
|
||||
if (lookupRequest.getVolumeIdsCount() > 0) {
|
||||
FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
|
||||
.getBlockingStub().lookupVolume(lookupRequest.build());
|
||||
Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
|
||||
for (Map.Entry<String,FilerProto.Locations> entry : vid2Locations.entrySet()) {
|
||||
for (Map.Entry<String, FilerProto.Locations> entry : vid2Locations.entrySet()) {
|
||||
volumeIdCache.setLocations(entry.getKey(), entry.getValue());
|
||||
knownLocations.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -57,7 +63,7 @@ public class SeaweedRead {
|
|||
startOffset += gap;
|
||||
}
|
||||
|
||||
FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId));
|
||||
FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId));
|
||||
if (locations == null || locations.getLocationsCount() == 0) {
|
||||
LOG.error("failed to locate {}", chunkView.fileId);
|
||||
// log here!
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ public class VolumeIdCache {
|
|||
}
|
||||
this.cache = CacheBuilder.newBuilder()
|
||||
.maximumSize(maxEntries)
|
||||
.expireAfterAccess(1, TimeUnit.HOURS)
|
||||
.expireAfterAccess(5, TimeUnit.MINUTES)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -362,6 +362,7 @@ message FilerConf {
|
|||
SSD = 1;
|
||||
}
|
||||
DiskType disk_type = 5;
|
||||
bool fsync = 6;
|
||||
}
|
||||
repeated PathConf locations = 2;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@
|
|||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
<version>1.5.4</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>1.5.3</version>
|
||||
<version>1.5.4</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.example.test;
|
||||
package com.seaweedfs.examples;
|
||||
|
||||
import seaweed.hdfs.SeaweedInputStream;
|
||||
import seaweedfs.client.FilerClient;
|
||||
|
|
@ -10,17 +10,20 @@ import java.io.InputStream;
|
|||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
public class Example {
|
||||
|
||||
public static FilerClient filerClient = new FilerClient("localhost", 18888);
|
||||
public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888);
|
||||
public class UnzipFile {
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
|
||||
// 本地模式，速度很快
|
||||
FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888);
|
||||
FilerClient filerClient = new FilerClient(filerGrpcClient);
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
parseZip("/Users/chris/tmp/test.zip");
|
||||
|
||||
// swfs读取，慢
|
||||
long startTime2 = System.currentTimeMillis();
|
||||
|
||||
long localProcessTime = startTime2 - startTime;
|
||||
|
||||
SeaweedInputStream seaweedInputStream = new SeaweedInputStream(
|
||||
filerGrpcClient,
|
||||
new org.apache.hadoop.fs.FileSystem.Statistics(""),
|
||||
|
|
@ -29,6 +32,11 @@ public class Example {
|
|||
);
|
||||
parseZip(seaweedInputStream);
|
||||
|
||||
long swProcessTime = System.currentTimeMillis() - startTime2;
|
||||
|
||||
System.out.println("Local time: " + localProcessTime);
|
||||
System.out.println("SeaweedFS time: " + swProcessTime);
|
||||
|
||||
}
|
||||
|
||||
public static void parseZip(String filename) throws IOException {
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package com.seaweedfs.examples;
|
||||
|
||||
import seaweedfs.client.FilerClient;
|
||||
import seaweedfs.client.FilerProto;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
|
||||
public class WatchFiles {
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
|
||||
FilerClient filerClient = new FilerClient("localhost", 18888);
|
||||
|
||||
long sinceNs = (System.currentTimeMillis() - 3600 * 1000) * 1000000L;
|
||||
|
||||
Iterator<FilerProto.SubscribeMetadataResponse> watch = filerClient.watch(
|
||||
"/buckets",
|
||||
"exampleClientName",
|
||||
sinceNs
|
||||
);
|
||||
|
||||
System.out.println("Connected to filer, subscribing from " + new Date());
|
||||
|
||||
while (watch.hasNext()) {
|
||||
FilerProto.SubscribeMetadataResponse event = watch.next();
|
||||
FilerProto.EventNotification notification = event.getEventNotification();
|
||||
if (!event.getDirectory().equals(notification.getNewParentPath())) {
|
||||
// move an entry to a new directory, possibly with a new name
|
||||
if (notification.hasOldEntry() && notification.hasNewEntry()) {
|
||||
System.out.println("moved " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName());
|
||||
} else {
|
||||
System.out.println("this should not happen.");
|
||||
}
|
||||
} else if (notification.hasNewEntry() && !notification.hasOldEntry()) {
|
||||
System.out.println("created entry " + event.getDirectory() + "/" + notification.getNewEntry().getName());
|
||||
} else if (!notification.hasNewEntry() && notification.hasOldEntry()) {
|
||||
System.out.println("deleted entry " + event.getDirectory() + "/" + notification.getOldEntry().getName());
|
||||
} else if (notification.hasNewEntry() && notification.hasOldEntry()) {
|
||||
System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -301,7 +301,7 @@
|
|||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.5.4</seaweedfs.client.version>
|
||||
<hadoop.version>2.9.2</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.5.4</seaweedfs.client.version>
|
||||
<hadoop.version>2.9.2</hadoop.version>
|
||||
</properties>
|
||||
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@
|
|||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.5.4</seaweedfs.client.version>
|
||||
<hadoop.version>3.1.1</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>1.5.3</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>1.5.4</seaweedfs.client.version>
|
||||
<hadoop.version>3.1.1</hadoop.version>
|
||||
</properties>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ type Filer struct {
|
|||
metaLogReplication string
|
||||
MetaAggregator *MetaAggregator
|
||||
Signature int32
|
||||
FilerConf *FilerConf
|
||||
}
|
||||
|
||||
func NewFiler(masters []string, grpcDialOption grpc.DialOption,
|
||||
|
|
@ -49,6 +50,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption,
|
|||
MasterClient: wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, dataCenter, masters),
|
||||
fileIdDeletionQueue: util.NewUnboundedQueue(),
|
||||
GrpcDialOption: grpcDialOption,
|
||||
FilerConf: NewFilerConf(),
|
||||
}
|
||||
f.LocalMetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, f.logFlushFunc, notifyFn)
|
||||
f.metaLogCollection = collection
|
||||
|
|
|
|||
139
weed/filer/filer_conf.go
Normal file
139
weed/filer/filer_conf.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
"github.com/golang/protobuf/jsonpb"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
const (
|
||||
DirectoryEtc = "/etc"
|
||||
FilerConfName = "filer.conf"
|
||||
)
|
||||
|
||||
type FilerConf struct {
|
||||
rules ptrie.Trie
|
||||
}
|
||||
|
||||
func NewFilerConf() (fc *FilerConf) {
|
||||
fc = &FilerConf{
|
||||
rules: ptrie.New(),
|
||||
}
|
||||
return fc
|
||||
}
|
||||
|
||||
func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) {
|
||||
filerConfPath := util.NewFullPath(DirectoryEtc, FilerConfName)
|
||||
entry, err := filer.FindEntry(context.Background(), filerConfPath)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
return nil
|
||||
}
|
||||
glog.Errorf("read filer conf entry %s: %v", filerConfPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
return fc.loadFromChunks(filer, entry.Chunks)
|
||||
}
|
||||
|
||||
func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) (err error) {
|
||||
data, err := filer.readEntry(chunks)
|
||||
if err != nil {
|
||||
glog.Errorf("read filer conf content: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
return fc.LoadFromBytes(data)
|
||||
}
|
||||
|
||||
func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
|
||||
conf := &filer_pb.FilerConf{}
|
||||
|
||||
if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil {
|
||||
|
||||
err = proto.UnmarshalText(string(data), conf)
|
||||
if err != nil {
|
||||
glog.Errorf("unable to parse filer conf: %v", err)
|
||||
// this is not recoverable
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return fc.doLoadConf(conf)
|
||||
}
|
||||
|
||||
func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) {
|
||||
for _, location := range conf.Locations {
|
||||
err = fc.AddLocationConf(location)
|
||||
if err != nil {
|
||||
// this is not recoverable
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) {
|
||||
err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf)
|
||||
if err != nil {
|
||||
glog.Errorf("put location prefix: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
|
||||
rules := ptrie.New()
|
||||
fc.rules.Walk(func(key []byte, value interface{}) bool {
|
||||
if string(key) == locationPrefix {
|
||||
return true
|
||||
}
|
||||
rules.Put(key, value)
|
||||
return true
|
||||
})
|
||||
fc.rules = rules
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
EmptyFilerConfPathConf = &filer_pb.FilerConf_PathConf{}
|
||||
)
|
||||
|
||||
func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf) {
|
||||
fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
|
||||
pathConf = value.(*filer_pb.FilerConf_PathConf)
|
||||
return true
|
||||
})
|
||||
if pathConf == nil {
|
||||
return EmptyFilerConfPathConf
|
||||
}
|
||||
return pathConf
|
||||
}
|
||||
|
||||
func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
|
||||
m := &filer_pb.FilerConf{}
|
||||
fc.rules.Walk(func(key []byte, value interface{}) bool {
|
||||
pathConf := value.(*filer_pb.FilerConf_PathConf)
|
||||
m.Locations = append(m.Locations, pathConf)
|
||||
return true
|
||||
})
|
||||
return m
|
||||
}
|
||||
|
||||
func (fc *FilerConf) ToText(writer io.Writer) error {
|
||||
|
||||
m := jsonpb.Marshaler{
|
||||
EmitDefaults: false,
|
||||
Indent: " ",
|
||||
}
|
||||
|
||||
return m.Marshal(writer, fc.ToProto())
|
||||
}
|
||||
29
weed/filer/filer_conf_test.go
Normal file
29
weed/filer/filer_conf_test.go
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestFilerConf(t *testing.T) {
|
||||
|
||||
fc := NewFilerConf()
|
||||
|
||||
conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{
|
||||
{
|
||||
LocationPrefix: "/buckets/abc",
|
||||
Collection: "abc",
|
||||
},
|
||||
{
|
||||
LocationPrefix: "/buckets/abcd",
|
||||
Collection: "abcd",
|
||||
},
|
||||
}}
|
||||
fc.doLoadConf(conf)
|
||||
|
||||
assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection)
|
||||
assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection)
|
||||
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@ import (
|
|||
|
||||
func (f *Filer) appendToFile(targetFile string, data []byte) error {
|
||||
|
||||
assignResult, uploadResult, err2 := f.assignAndUpload(data)
|
||||
assignResult, uploadResult, err2 := f.assignAndUpload(targetFile, data)
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
|
|
@ -46,14 +46,16 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
|
||||
func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
|
||||
// assign a volume location
|
||||
rule := f.FilerConf.MatchStorageRule(targetFile)
|
||||
assignRequest := &operation.VolumeAssignRequest{
|
||||
Count: 1,
|
||||
Collection: f.metaLogCollection,
|
||||
Replication: f.metaLogReplication,
|
||||
Collection: util.Nvl(f.metaLogCollection, rule.Collection),
|
||||
Replication: util.Nvl(f.metaLogReplication, rule.Replication),
|
||||
WritableVolumeCount: 1,
|
||||
}
|
||||
|
||||
assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("AssignVolume: %v", err)
|
||||
|
|
|
|||
61
weed/filer/filer_on_meta_event.go
Normal file
61
weed/filer/filer_on_meta_event.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
|
||||
func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
|
||||
if DirectoryEtc != event.Directory {
|
||||
if DirectoryEtc != event.EventNotification.NewParentPath {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
entry := event.EventNotification.NewEntry
|
||||
if entry == nil {
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(0).Infof("procesing %v", event)
|
||||
if entry.Name == FilerConfName {
|
||||
f.reloadFilerConfiguration(entry)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
|
||||
fc := NewFilerConf()
|
||||
err := fc.loadFromChunks(f, entry.Chunks)
|
||||
if err != nil {
|
||||
glog.Errorf("read filer conf chunks: %v", err)
|
||||
return
|
||||
}
|
||||
f.FilerConf = fc
|
||||
}
|
||||
|
||||
func (f *Filer) LoadFilerConf() {
|
||||
fc := NewFilerConf()
|
||||
err := util.Retry("loadFilerConf", func() error {
|
||||
return fc.loadFromFiler(f)
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("read filer conf: %v", err)
|
||||
return
|
||||
}
|
||||
f.FilerConf = fc
|
||||
}
|
||||
|
|
@ -141,6 +141,9 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
|
|||
return fmt.Errorf("process %v: %v", resp, err)
|
||||
}
|
||||
lastTsNs = resp.TsNs
|
||||
|
||||
f.onMetadataChangeEvent(resp)
|
||||
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -294,7 +294,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
|
|||
// find the earliest incoming chunk
|
||||
newChunks := chunks
|
||||
earliestChunk := newChunks[0]
|
||||
for i:=1;i<len(newChunks);i++{
|
||||
for i := 1; i < len(newChunks); i++ {
|
||||
if lessThan(earliestChunk, newChunks[i]) {
|
||||
earliestChunk = newChunks[i]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
|
||||
|
|
@ -101,3 +102,38 @@ func LookupJwt(master string, fileId string) security.EncodedJwt {
|
|||
|
||||
return security.EncodedJwt(tokenStr)
|
||||
}
|
||||
|
||||
type StorageOption struct {
|
||||
Replication string
|
||||
Collection string
|
||||
DataCenter string
|
||||
Rack string
|
||||
TtlSeconds int32
|
||||
Fsync bool
|
||||
}
|
||||
|
||||
func (so *StorageOption) TtlString() string {
|
||||
return needle.SecondsToTTL(so.TtlSeconds)
|
||||
}
|
||||
|
||||
func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, altRequest *VolumeAssignRequest) {
|
||||
ar = &VolumeAssignRequest{
|
||||
Count: uint64(count),
|
||||
Replication: so.Replication,
|
||||
Collection: so.Collection,
|
||||
Ttl: so.TtlString(),
|
||||
DataCenter: so.DataCenter,
|
||||
Rack: so.Rack,
|
||||
}
|
||||
if so.DataCenter != "" || so.Rack != "" {
|
||||
altRequest = &VolumeAssignRequest{
|
||||
Count: uint64(count),
|
||||
Replication: so.Replication,
|
||||
Collection: so.Collection,
|
||||
Ttl: so.TtlString(),
|
||||
DataCenter: "",
|
||||
Rack: "",
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -362,6 +362,7 @@ message FilerConf {
|
|||
SSD = 1;
|
||||
}
|
||||
DiskType disk_type = 5;
|
||||
bool fsync = 6;
|
||||
}
|
||||
repeated PathConf locations = 2;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3171,6 +3171,7 @@ type FilerConf_PathConf struct {
|
|||
Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"`
|
||||
Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"`
|
||||
DiskType FilerConf_PathConf_DiskType `protobuf:"varint,5,opt,name=disk_type,json=diskType,proto3,enum=filer_pb.FilerConf_PathConf_DiskType" json:"disk_type,omitempty"`
|
||||
Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"`
|
||||
}
|
||||
|
||||
func (x *FilerConf_PathConf) Reset() {
|
||||
|
|
@ -3240,6 +3241,13 @@ func (x *FilerConf_PathConf) GetDiskType() FilerConf_PathConf_DiskType {
|
|||
return FilerConf_PathConf_HDD
|
||||
}
|
||||
|
||||
func (x *FilerConf_PathConf) GetFsync() bool {
|
||||
if x != nil {
|
||||
return x.Fsync
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_filer_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_filer_proto_rawDesc = []byte{
|
||||
|
|
@ -3614,13 +3622,13 @@ var file_filer_proto_rawDesc = []byte{
|
|||
0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c,
|
||||
0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xcd, 0x02, 0x0a, 0x09, 0x46, 0x69,
|
||||
0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xe3, 0x02, 0x0a, 0x09, 0x46, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
|
||||
0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
|
||||
0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02,
|
||||
0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f,
|
||||
0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xe9, 0x01,
|
||||
0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xff, 0x01,
|
||||
0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f,
|
||||
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65,
|
||||
|
|
@ -3633,117 +3641,118 @@ var file_filer_proto_rawDesc = []byte{
|
|||
0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x66, 0x69, 0x6c,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e,
|
||||
0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70,
|
||||
0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x1c, 0x0a, 0x08, 0x44,
|
||||
0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x00,
|
||||
0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x01, 0x32, 0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65,
|
||||
0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f,
|
||||
0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74,
|
||||
0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f,
|
||||
0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74,
|
||||
0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63,
|
||||
0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69,
|
||||
0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74,
|
||||
0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74,
|
||||
0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72,
|
||||
0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61,
|
||||
0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79,
|
||||
0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61,
|
||||
0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
|
||||
0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
|
||||
0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79,
|
||||
0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65,
|
||||
0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65,
|
||||
0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74,
|
||||
0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65,
|
||||
0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65,
|
||||
0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d,
|
||||
0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e,
|
||||
0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61,
|
||||
0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
|
||||
0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73,
|
||||
0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69,
|
||||
0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
|
||||
0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f,
|
||||
0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f,
|
||||
0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65,
|
||||
0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
|
||||
0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c,
|
||||
0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69,
|
||||
0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74,
|
||||
0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f,
|
||||
0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43,
|
||||
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47,
|
||||
0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60,
|
||||
0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64,
|
||||
0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53,
|
||||
0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61,
|
||||
0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01,
|
||||
0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63,
|
||||
0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d,
|
||||
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72,
|
||||
0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x56, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43,
|
||||
0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65,
|
||||
0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65,
|
||||
0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12,
|
||||
0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12,
|
||||
0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74,
|
||||
0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65,
|
||||
0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
|
||||
0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47,
|
||||
0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05,
|
||||
0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e,
|
||||
0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77,
|
||||
0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
|
||||
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73,
|
||||
0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
|
||||
0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x33,
|
||||
0x65, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66,
|
||||
0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e,
|
||||
0x63, 0x22, 0x1c, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a,
|
||||
0x03, 0x48, 0x44, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x01, 0x32,
|
||||
0xdc, 0x0c, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
|
||||
0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
|
||||
0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
|
||||
0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73,
|
||||
0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65,
|
||||
0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74,
|
||||
0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54,
|
||||
0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c,
|
||||
0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69,
|
||||
0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66,
|
||||
0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65,
|
||||
0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d,
|
||||
0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67,
|
||||
0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b,
|
||||
0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c,
|
||||
0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66,
|
||||
0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
|
||||
0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a,
|
||||
0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63,
|
||||
0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46,
|
||||
0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74,
|
||||
0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e,
|
||||
0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62,
|
||||
0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65,
|
||||
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e,
|
||||
0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
|
||||
0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72,
|
||||
0x69, 0x62, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
|
||||
0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73,
|
||||
0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
|
||||
0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x56, 0x0a,
|
||||
0x0d, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1e,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
|
||||
0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f,
|
||||
0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
|
||||
0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42,
|
||||
0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x12,
|
||||
0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69,
|
||||
0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b,
|
||||
0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f,
|
||||
0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65,
|
||||
0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f,
|
||||
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73,
|
||||
0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77,
|
||||
0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62,
|
||||
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import (
|
|||
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/viant/ptrie"
|
||||
)
|
||||
|
||||
func ToFileIdObject(fileIdStr string) (*FileId, error) {
|
||||
|
|
@ -121,3 +123,27 @@ func LookupEntry(client SeaweedFilerClient, request *LookupDirectoryEntryRequest
|
|||
}
|
||||
|
||||
var ErrNotFound = errors.New("filer: no entry is found in filer store")
|
||||
|
||||
func IsCreate(event *SubscribeMetadataResponse) bool {
|
||||
return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry == nil
|
||||
}
|
||||
func IsUpdate(event *SubscribeMetadataResponse) bool {
|
||||
return event.EventNotification.NewEntry != nil &&
|
||||
event.EventNotification.OldEntry != nil &&
|
||||
event.Directory == event.EventNotification.NewParentPath
|
||||
}
|
||||
func IsDelete(event *SubscribeMetadataResponse) bool {
|
||||
return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry != nil
|
||||
}
|
||||
func IsRename(event *SubscribeMetadataResponse) bool {
|
||||
return event.EventNotification.NewEntry != nil &&
|
||||
event.EventNotification.OldEntry != nil &&
|
||||
event.Directory != event.EventNotification.NewParentPath
|
||||
}
|
||||
|
||||
var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
|
||||
|
||||
func (fp *FilerConf_PathConf) Key() interface{} {
|
||||
key, _ := proto.Marshal(fp)
|
||||
return string(key)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,5 +32,5 @@ const (
|
|||
// Non-Standard S3 HTTP request constants
|
||||
const (
|
||||
AmzIdentityId = "x-amz-identity-id"
|
||||
AmzIsAdmin = "x-amz-is-admin" // only set to http request header as a context
|
||||
AmzIsAdmin = "x-amz-is-admin" // only set to http request header as a context
|
||||
)
|
||||
|
|
|
|||
|
|
@ -118,18 +118,12 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
|
|||
|
||||
bucket, _ := getBucketAndObject(r)
|
||||
|
||||
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if entry == nil || err == filer_pb.ErrNotFound {
|
||||
writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
|
||||
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
|
||||
writeErrorResponse(w, err, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !s3a.hasAccess(r, entry) {
|
||||
writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
// delete collection
|
||||
deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
|
||||
|
|
@ -158,20 +152,26 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
|
|||
|
||||
bucket, _ := getBucketAndObject(r)
|
||||
|
||||
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if entry == nil || err == filer_pb.ErrNotFound {
|
||||
writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !s3a.hasAccess(r, entry) {
|
||||
writeErrorResponse(w, s3err.ErrAccessDenied, r.URL)
|
||||
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
|
||||
writeErrorResponse(w, err, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseEmpty(w)
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
|
||||
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if entry == nil || err == filer_pb.ErrNotFound {
|
||||
return s3err.ErrNoSuchBucket
|
||||
}
|
||||
|
||||
if !s3a.hasAccess(r, entry) {
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
|
||||
isAdmin := r.Header.Get(xhttp.AmzIsAdmin) != ""
|
||||
if isAdmin {
|
||||
|
|
@ -188,4 +188,4 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
|
|||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
|
|||
ar := &operation.VolumeAssignRequest{
|
||||
Count: count,
|
||||
DataCenter: r.FormValue("dataCenter"),
|
||||
Rack: r.FormValue("rack"),
|
||||
Replication: r.FormValue("replication"),
|
||||
Collection: r.FormValue("collection"),
|
||||
Ttl: r.FormValue("ttl"),
|
||||
|
|
|
|||
|
|
@ -156,7 +156,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
|
|||
|
||||
resp = &filer_pb.CreateEntryResponse{}
|
||||
|
||||
chunks, garbage, err2 := fs.cleanupChunks(nil, req.Entry)
|
||||
chunks, garbage, err2 := fs.cleanupChunks(util.Join(req.Directory, req.Entry.Name), nil, req.Entry)
|
||||
if err2 != nil {
|
||||
return &filer_pb.CreateEntryResponse{}, fmt.Errorf("CreateEntry cleanupChunks %s %s: %v", req.Directory, req.Entry.Name, err2)
|
||||
}
|
||||
|
|
@ -190,7 +190,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
|
|||
return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)
|
||||
}
|
||||
|
||||
chunks, garbage, err2 := fs.cleanupChunks(entry, req.Entry)
|
||||
chunks, garbage, err2 := fs.cleanupChunks(fullpath, entry, req.Entry)
|
||||
if err2 != nil {
|
||||
return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("UpdateEntry cleanupChunks %s: %v", fullpath, err2)
|
||||
}
|
||||
|
|
@ -240,7 +240,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
|
|||
return &filer_pb.UpdateEntryResponse{}, err
|
||||
}
|
||||
|
||||
func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) {
|
||||
func (fs *FilerServer) cleanupChunks(fullpath string, existingEntry *filer.Entry, newEntry *filer_pb.Entry) (chunks, garbage []*filer_pb.FileChunk, err error) {
|
||||
|
||||
// remove old chunks if not included in the new ones
|
||||
if existingEntry != nil {
|
||||
|
|
@ -257,7 +257,14 @@ func (fs *FilerServer) cleanupChunks(existingEntry *filer.Entry, newEntry *filer
|
|||
garbage = append(garbage, coveredChunks...)
|
||||
|
||||
if newEntry.Attributes != nil {
|
||||
chunks, err = filer.MaybeManifestize(fs.saveAsChunk(newEntry.Attributes.Replication, newEntry.Attributes.Collection, "", "", needle.SecondsToTTL(newEntry.Attributes.TtlSec), false), chunks)
|
||||
so := fs.detectStorageOption(fullpath,
|
||||
newEntry.Attributes.Collection,
|
||||
newEntry.Attributes.Replication,
|
||||
newEntry.Attributes.TtlSec,
|
||||
"",
|
||||
"",
|
||||
)
|
||||
chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), chunks)
|
||||
if err != nil {
|
||||
// not good, but should be ok
|
||||
glog.V(0).Infof("MaybeManifestize: %v", err)
|
||||
|
|
@ -275,7 +282,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
|
|||
|
||||
fullpath := util.NewFullPath(req.Directory, req.EntryName)
|
||||
var offset int64 = 0
|
||||
entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))
|
||||
entry, err := fs.filer.FindEntry(ctx, fullpath)
|
||||
if err == filer_pb.ErrNotFound {
|
||||
entry = &filer.Entry{
|
||||
FullPath: fullpath,
|
||||
|
|
@ -297,8 +304,8 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
|
|||
}
|
||||
|
||||
entry.Chunks = append(entry.Chunks, req.Chunks...)
|
||||
|
||||
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(entry.Replication, entry.Collection, "", "", needle.SecondsToTTL(entry.TtlSec), false), entry.Chunks)
|
||||
so := fs.detectStorageOption(string(fullpath), entry.Collection, entry.Replication, entry.TtlSec, "", "")
|
||||
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.Chunks)
|
||||
if err != nil {
|
||||
// not good, but should be ok
|
||||
glog.V(0).Infof("MaybeManifestize: %v", err)
|
||||
|
|
@ -323,41 +330,10 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
|
|||
|
||||
func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
|
||||
|
||||
ttlStr := ""
|
||||
if req.TtlSec > 0 {
|
||||
ttlStr = strconv.Itoa(int(req.TtlSec))
|
||||
}
|
||||
collection, replication, _ := fs.detectCollection(req.Path, req.Collection, req.Replication)
|
||||
so := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DataCenter, req.Rack)
|
||||
|
||||
var altRequest *operation.VolumeAssignRequest
|
||||
assignRequest, altRequest := so.ToAssignRequests(int(req.Count))
|
||||
|
||||
dataCenter := req.DataCenter
|
||||
if dataCenter == "" {
|
||||
dataCenter = fs.option.DataCenter
|
||||
}
|
||||
rack := req.Rack
|
||||
if rack == "" {
|
||||
rack = fs.option.Rack
|
||||
}
|
||||
|
||||
assignRequest := &operation.VolumeAssignRequest{
|
||||
Count: uint64(req.Count),
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
Ttl: ttlStr,
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
}
|
||||
if dataCenter != "" || rack != "" {
|
||||
altRequest = &operation.VolumeAssignRequest{
|
||||
Count: uint64(req.Count),
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
Ttl: ttlStr,
|
||||
DataCenter: "",
|
||||
Rack: "",
|
||||
}
|
||||
}
|
||||
assignResult, err := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, assignRequest, altRequest)
|
||||
if err != nil {
|
||||
glog.V(3).Infof("AssignVolume: %v", err)
|
||||
|
|
@ -374,8 +350,8 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
|
|||
Url: assignResult.Url,
|
||||
PublicUrl: assignResult.PublicUrl,
|
||||
Auth: string(assignResult.Auth),
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
Collection: so.Collection,
|
||||
Replication: so.Replication,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
|
|||
|
||||
fs.filer.LoadBuckets()
|
||||
|
||||
fs.filer.LoadFilerConf()
|
||||
|
||||
grace.OnInterrupt(func() {
|
||||
fs.filer.Shutdown()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
|||
case "DELETE":
|
||||
stats.FilerRequestCounter.WithLabelValues("delete").Inc()
|
||||
if _, ok := r.URL.Query()["tagging"]; ok {
|
||||
fs.DeleteTaggingHandler(w,r)
|
||||
fs.DeleteTaggingHandler(w, r)
|
||||
} else {
|
||||
fs.DeleteHandler(w, r)
|
||||
}
|
||||
|
|
@ -35,7 +35,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
|||
case "PUT":
|
||||
stats.FilerRequestCounter.WithLabelValues("put").Inc()
|
||||
if _, ok := r.URL.Query()["tagging"]; ok {
|
||||
fs.PutTaggingHandler(w,r)
|
||||
fs.PutTaggingHandler(w, r)
|
||||
} else {
|
||||
fs.PostHandler(w, r)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,30 +29,13 @@ type FilerPostResult struct {
|
|||
Url string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
func (fs *FilerServer) assignNewFileInfo(replication, collection, dataCenter, rack, ttlString string, fsync bool) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
|
||||
func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues("assign").Inc()
|
||||
start := time.Now()
|
||||
defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }()
|
||||
|
||||
ar := &operation.VolumeAssignRequest{
|
||||
Count: 1,
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
Ttl: ttlString,
|
||||
DataCenter: dataCenter,
|
||||
}
|
||||
var altRequest *operation.VolumeAssignRequest
|
||||
if dataCenter != "" || rack != "" {
|
||||
altRequest = &operation.VolumeAssignRequest{
|
||||
Count: 1,
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
Ttl: ttlString,
|
||||
DataCenter: "",
|
||||
Rack: "",
|
||||
}
|
||||
}
|
||||
ar, altRequest := so.ToAssignRequests(1)
|
||||
|
||||
assignResult, ae := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, ar, altRequest)
|
||||
if ae != nil {
|
||||
|
|
@ -62,7 +45,7 @@ func (fs *FilerServer) assignNewFileInfo(replication, collection, dataCenter, ra
|
|||
}
|
||||
fileId = assignResult.Fid
|
||||
urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
|
||||
if fsync {
|
||||
if so.Fsync {
|
||||
urlLocation += "?fsync=true"
|
||||
}
|
||||
auth = assignResult.Auth
|
||||
|
|
@ -74,25 +57,15 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
|
|||
ctx := context.Background()
|
||||
|
||||
query := r.URL.Query()
|
||||
collection, replication, fsync := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication"))
|
||||
dataCenter := query.Get("dataCenter")
|
||||
if dataCenter == "" {
|
||||
dataCenter = fs.option.DataCenter
|
||||
}
|
||||
rack := query.Get("rack")
|
||||
if dataCenter == "" {
|
||||
rack = fs.option.Rack
|
||||
}
|
||||
ttlString := r.URL.Query().Get("ttl")
|
||||
so := fs.detectStorageOption0(r.RequestURI,
|
||||
query.Get("collection"),
|
||||
query.Get("replication"),
|
||||
query.Get("ttl"),
|
||||
query.Get("dataCenter"),
|
||||
query.Get("rack"),
|
||||
)
|
||||
|
||||
// read ttl in seconds
|
||||
ttl, err := needle.ReadTTL(ttlString)
|
||||
ttlSeconds := int32(0)
|
||||
if err == nil {
|
||||
ttlSeconds = int32(ttl.Minutes()) * 60
|
||||
}
|
||||
|
||||
fs.autoChunk(ctx, w, r, replication, collection, dataCenter, rack, ttlSeconds, ttlString, fsync)
|
||||
fs.autoChunk(ctx, w, r, so)
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -130,21 +103,12 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string, fsync bool) {
|
||||
// default
|
||||
collection = fs.option.Collection
|
||||
replication = fs.option.DefaultReplication
|
||||
|
||||
// get default collection settings
|
||||
if qCollection != "" {
|
||||
collection = qCollection
|
||||
}
|
||||
if qReplication != "" {
|
||||
replication = qReplication
|
||||
}
|
||||
func (fs *FilerServer) detectStorageOption(requestURI, qCollection, qReplication string, ttlSeconds int32, dataCenter, rack string) *operation.StorageOption {
|
||||
collection := util.Nvl(qCollection, fs.option.Collection)
|
||||
replication := util.Nvl(qReplication, fs.option.DefaultReplication)
|
||||
|
||||
// required by buckets folder
|
||||
bucketDefaultReplication := ""
|
||||
bucketDefaultReplication, fsync := "", false
|
||||
if strings.HasPrefix(requestURI, fs.filer.DirBucketsPath+"/") {
|
||||
bucketAndObjectKey := requestURI[len(fs.filer.DirBucketsPath)+1:]
|
||||
t := strings.Index(bucketAndObjectKey, "/")
|
||||
|
|
@ -160,5 +124,32 @@ func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication st
|
|||
replication = bucketDefaultReplication
|
||||
}
|
||||
|
||||
return
|
||||
rule := fs.filer.FilerConf.MatchStorageRule(requestURI)
|
||||
|
||||
if ttlSeconds == 0 {
|
||||
ttl, err := needle.ReadTTL(rule.GetTtl())
|
||||
if err != nil {
|
||||
glog.Errorf("fail to parse %s ttl setting %s: %v", rule.LocationPrefix, rule.Ttl, err)
|
||||
}
|
||||
ttlSeconds = int32(ttl.Minutes()) * 60
|
||||
}
|
||||
|
||||
return &operation.StorageOption{
|
||||
Replication: util.Nvl(replication, rule.Replication),
|
||||
Collection: util.Nvl(collection, rule.Collection),
|
||||
DataCenter: util.Nvl(dataCenter, fs.option.DataCenter),
|
||||
Rack: util.Nvl(rack, fs.option.Rack),
|
||||
TtlSeconds: ttlSeconds,
|
||||
Fsync: fsync || rule.Fsync,
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FilerServer) detectStorageOption0(requestURI, qCollection, qReplication string, qTtl string, dataCenter, rack string) *operation.StorageOption {
|
||||
|
||||
ttl, err := needle.ReadTTL(qTtl)
|
||||
if err != nil {
|
||||
glog.Errorf("fail to parse ttl %s: %v", qTtl, err)
|
||||
}
|
||||
|
||||
return fs.detectStorageOption(requestURI, qCollection, qReplication, int32(ttl.Minutes())*60, dataCenter, rack)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import (
|
|||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) {
|
||||
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
|
||||
|
||||
// autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
|
||||
query := r.URL.Query()
|
||||
|
|
@ -51,10 +51,10 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
|||
if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") {
|
||||
reply, err = fs.mkdir(ctx, w, r)
|
||||
} else {
|
||||
reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, replication, collection, dataCenter, rack, ttlSec, ttlString, fsync)
|
||||
reply, md5bytes, err = fs.doPostAutoChunk(ctx, w, r, chunkSize, so)
|
||||
}
|
||||
} else {
|
||||
reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, replication, collection, dataCenter, rack, ttlSec, ttlString, fsync)
|
||||
reply, md5bytes, err = fs.doPutAutoChunk(ctx, w, r, chunkSize, so)
|
||||
}
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
|
|
@ -66,7 +66,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
|||
}
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
|
||||
multipartReader, multipartReaderErr := r.MultipartReader()
|
||||
if multipartReaderErr != nil {
|
||||
|
|
@ -87,46 +87,46 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
|
|||
contentType = ""
|
||||
}
|
||||
|
||||
fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, part1, chunkSize, replication, collection, dataCenter, rack, ttlString, fileName, contentType, fsync)
|
||||
fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, part1, chunkSize, fileName, contentType, so)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, rack, ttlString, fsync), fileChunks)
|
||||
fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
|
||||
if replyerr != nil {
|
||||
glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
|
||||
return
|
||||
}
|
||||
|
||||
md5bytes = md5Hash.Sum(nil)
|
||||
filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, replication, collection, ttlSec, contentType, md5bytes, fileChunks, chunkOffset)
|
||||
filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlSec int32, ttlString string, fsync bool) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, chunkSize int32, so *operation.StorageOption) (filerResult *FilerPostResult, md5bytes []byte, replyerr error) {
|
||||
|
||||
fileName := ""
|
||||
contentType := ""
|
||||
|
||||
fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, replication, collection, dataCenter, rack, ttlString, fileName, contentType, fsync)
|
||||
fileChunks, md5Hash, chunkOffset, err := fs.uploadReaderToChunks(w, r, r.Body, chunkSize, fileName, contentType, so)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(replication, collection, dataCenter, rack, ttlString, fsync), fileChunks)
|
||||
fileChunks, replyerr = filer.MaybeManifestize(fs.saveAsChunk(so), fileChunks)
|
||||
if replyerr != nil {
|
||||
glog.V(0).Infof("manifestize %s: %v", r.RequestURI, replyerr)
|
||||
return
|
||||
}
|
||||
|
||||
md5bytes = md5Hash.Sum(nil)
|
||||
filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, replication, collection, ttlSec, contentType, md5bytes, fileChunks, chunkOffset)
|
||||
filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, replication string, collection string, ttlSec int32, contentType string, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) {
|
||||
func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64) (filerResult *FilerPostResult, replyerr error) {
|
||||
|
||||
// detect file mode
|
||||
modeStr := r.URL.Query().Get("mode")
|
||||
|
|
@ -163,9 +163,9 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
Mode: os.FileMode(mode),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
TtlSec: ttlSec,
|
||||
Replication: so.Replication,
|
||||
Collection: so.Collection,
|
||||
TtlSec: so.TtlSeconds,
|
||||
Mime: contentType,
|
||||
Md5: md5bytes,
|
||||
FileSize: uint64(chunkOffset),
|
||||
|
|
@ -199,7 +199,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
|||
return filerResult, replyerr
|
||||
}
|
||||
|
||||
func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, replication string, collection string, dataCenter string, rack string, ttlString string, fileName string, contentType string, fsync bool) ([]*filer_pb.FileChunk, hash.Hash, int64, error) {
|
||||
func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error) {
|
||||
var fileChunks []*filer_pb.FileChunk
|
||||
|
||||
md5Hash := md5.New()
|
||||
|
|
@ -211,7 +211,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
|
|||
limitedReader := io.LimitReader(partReader, int64(chunkSize))
|
||||
|
||||
// assign one file id for one chunk
|
||||
fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync)
|
||||
fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
|
||||
if assignErr != nil {
|
||||
return nil, nil, 0, assignErr
|
||||
}
|
||||
|
|
@ -255,11 +255,11 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht
|
|||
return uploadResult, err
|
||||
}
|
||||
|
||||
func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCenter string, rack string, ttlString string, fsync bool) filer.SaveDataAsChunkFunctionType {
|
||||
func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
|
||||
|
||||
return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, string, string, error) {
|
||||
// assign one file id for one chunk
|
||||
fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync)
|
||||
fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
|
||||
if assignErr != nil {
|
||||
return nil, "", "", assignErr
|
||||
}
|
||||
|
|
@ -270,7 +270,7 @@ func (fs *FilerServer) saveAsChunk(replication string, collection string, dataCe
|
|||
return nil, "", "", uploadErr
|
||||
}
|
||||
|
||||
return uploadResult.ToPbFileChunk(fileId, offset), collection, replication, nil
|
||||
return uploadResult.ToPbFileChunk(fileId, offset), so.Collection, so.Replication, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ import (
|
|||
)
|
||||
|
||||
// handling single chunk POST or PUT upload
|
||||
func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, replication string, collection string, dataCenter string, rack string, ttlSeconds int32, ttlString string, fsync bool) (filerResult *FilerPostResult, err error) {
|
||||
func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) (filerResult *FilerPostResult, err error) {
|
||||
|
||||
fileId, urlLocation, auth, err := fs.assignNewFileInfo(replication, collection, dataCenter, rack, ttlString, fsync)
|
||||
fileId, urlLocation, auth, err := fs.assignNewFileInfo(so)
|
||||
|
||||
if err != nil || fileId == "" || urlLocation == "" {
|
||||
return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
|
||||
return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, so.Collection, so.DataCenter)
|
||||
}
|
||||
|
||||
glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation)
|
||||
|
|
@ -65,9 +65,9 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
|
|||
Mode: 0660,
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
Replication: replication,
|
||||
Collection: collection,
|
||||
TtlSec: ttlSeconds,
|
||||
Replication: so.Replication,
|
||||
Collection: so.Collection,
|
||||
TtlSec: so.TtlSeconds,
|
||||
Mime: pu.MimeType,
|
||||
Md5: util.Base64Md5ToBytes(pu.ContentMd5),
|
||||
},
|
||||
|
|
|
|||
126
weed/shell/command_fs_configure.go
Normal file
126
weed/shell/command_fs_configure.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package shell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/weed/filer"
|
||||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandFsConfigure{})
|
||||
}
|
||||
|
||||
type commandFsConfigure struct {
|
||||
}
|
||||
|
||||
func (c *commandFsConfigure) Name() string {
|
||||
return "fs.configure"
|
||||
}
|
||||
|
||||
func (c *commandFsConfigure) Help() string {
|
||||
return `configure and apply storage options for each location
|
||||
|
||||
# see the possible configuration file content
|
||||
fs.configure
|
||||
|
||||
# trying the changes and see the possible configuration file content
|
||||
fs.configure -locationPrfix=/my/folder -collection=abc
|
||||
fs.configure -locationPrfix=/my/folder -collection=abc -ttl=7d
|
||||
|
||||
# apply the changes
|
||||
fs.configure -locationPrfix=/my/folder -collection=abc -apply
|
||||
|
||||
# delete the changes
|
||||
fs.configure -locationPrfix=/my/folder -delete -apply
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
|
||||
fsConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix, required to update the path-specific configuration")
|
||||
collection := fsConfigureCommand.String("collection", "", "assign writes to this collection")
|
||||
replication := fsConfigureCommand.String("replication", "", "assign writes with this replication")
|
||||
ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl")
|
||||
fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes")
|
||||
isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix")
|
||||
apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration")
|
||||
if err = fsConfigureCommand.Parse(args); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
request := &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: filer.DirectoryEtc,
|
||||
Name: filer.FilerConfName,
|
||||
}
|
||||
respLookupEntry, err := filer_pb.LookupEntry(client, request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return filer.StreamContent(commandEnv.MasterClient, &buf, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
|
||||
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fc := filer.NewFilerConf()
|
||||
if err = fc.LoadFromBytes(buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if *locationPrefix != "" {
|
||||
locConf := &filer_pb.FilerConf_PathConf{
|
||||
LocationPrefix: *locationPrefix,
|
||||
Collection: *collection,
|
||||
Replication: *replication,
|
||||
Ttl: *ttl,
|
||||
Fsync: *fsync,
|
||||
}
|
||||
if *isDelete {
|
||||
fc.DeleteLocationConf(*locationPrefix)
|
||||
} else {
|
||||
fc.AddLocationConf(locConf)
|
||||
}
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
fc.ToText(&buf)
|
||||
|
||||
fmt.Fprintf(writer, string(buf.Bytes()))
|
||||
fmt.Fprintln(writer)
|
||||
|
||||
if *apply {
|
||||
|
||||
target := fmt.Sprintf("http://%s:%d%s/%s", commandEnv.option.FilerHost, commandEnv.option.FilerPort, filer.DirectoryEtc, filer.FilerConfName)
|
||||
|
||||
// set the HTTP method, url, and request body
|
||||
req, err := http.NewRequest(http.MethodPut, target, &buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set the request header Content-Type for json
|
||||
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
util.CloseResponse(resp)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 10)
|
||||
VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 11)
|
||||
COMMIT = ""
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
|
@ -29,22 +28,6 @@ func init() {
|
|||
}
|
||||
}
|
||||
|
||||
func PostBytes(url string, body []byte) ([]byte, error) {
|
||||
r, err := client.Post(url, "", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Post to %s: %v", url, err)
|
||||
}
|
||||
defer r.Body.Close()
|
||||
b, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Read response body: %v", err)
|
||||
}
|
||||
if r.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%s: %s", url, r.Status)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func Post(url string, values url.Values) ([]byte, error) {
|
||||
r, err := client.PostForm(url, values)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -29,3 +29,13 @@ func Retry(name string, job func() error) (err error) {
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// return the first non empty string
|
||||
func Nvl(values ...string) string {
|
||||
for _, s := range values {
|
||||
if s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue