mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Java 1.6.7 Support Mounted Remote Storage
Chris Lu
parent
5677fbc0a1
commit
994c231cf4
|
|
@ -26,7 +26,7 @@ Then get the seaweedfs hadoop client jar.
|
|||
|
||||
```
|
||||
cd share/hadoop/common/lib/
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.6.6/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
wget https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.6.7/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
```
|
||||
|
||||
# TestDFSIO Benchmark
|
||||
|
|
|
|||
|
|
@ -10,12 +10,12 @@ $ mvn install
|
|||
# build for hadoop2
|
||||
$cd $GOPATH/src/github.com/chrislusf/seaweedfs/other/java/hdfs2
|
||||
$ mvn package
|
||||
$ ls -al target/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
$ ls -al target/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
|
||||
# build for hadoop3
|
||||
$cd $GOPATH/src/github.com/chrislusf/seaweedfs/other/java/hdfs3
|
||||
$ mvn package
|
||||
$ ls -al target/seaweedfs-hadoop3-client-1.6.6.jar
|
||||
$ ls -al target/seaweedfs-hadoop3-client-1.6.7.jar
|
||||
|
||||
```
|
||||
Maven
|
||||
|
|
@ -23,7 +23,7 @@ Maven
|
|||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop3-client</artifactId>
|
||||
<version>1.6.6</version>
|
||||
<version>1.6.7</version>
|
||||
</dependency>
|
||||
|
||||
or
|
||||
|
|
@ -31,23 +31,23 @@ or
|
|||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>1.6.6</version>
|
||||
<version>1.6.7</version>
|
||||
</dependency>
|
||||
|
||||
```
|
||||
|
||||
Or you can download the latest version from MavenCentral
|
||||
* https://mvnrepository.com/artifact/com.github.chrislusf/seaweedfs-hadoop2-client
|
||||
* [seaweedfs-hadoop2-client-1.6.6.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.6.6/seaweedfs-hadoop2-client-1.6.6.jar)
|
||||
* [seaweedfs-hadoop2-client-1.6.7.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.6.7/seaweedfs-hadoop2-client-1.6.7.jar)
|
||||
* https://mvnrepository.com/artifact/com.github.chrislusf/seaweedfs-hadoop3-client
|
||||
* [seaweedfs-hadoop3-client-1.6.6.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop3-client/1.6.6/seaweedfs-hadoop3-client-1.6.6.jar)
|
||||
* [seaweedfs-hadoop3-client-1.6.7.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop3-client/1.6.7/seaweedfs-hadoop3-client-1.6.7.jar)
|
||||
|
||||
# Test SeaweedFS on Hadoop
|
||||
|
||||
Suppose you are getting a new Hadoop installation. Here are the minimum steps to get SeaweedFS to run.
|
||||
|
||||
You would need to start a weed filer first, build the seaweedfs-hadoop2-client-1.6.6.jar
|
||||
or seaweedfs-hadoop3-client-1.6.6.jar, and do the following:
|
||||
You would need to start a weed filer first, build the seaweedfs-hadoop2-client-1.6.7.jar
|
||||
or seaweedfs-hadoop3-client-1.6.7.jar, and do the following:
|
||||
|
||||
```
|
||||
# optionally adjust hadoop memory allocation
|
||||
|
|
@ -60,12 +60,12 @@ $ echo "<configuration></configuration>" > etc/hadoop/mapred-site.xml
|
|||
# on hadoop2
|
||||
$ bin/hdfs dfs -Dfs.defaultFS=seaweedfs://localhost:8888 \
|
||||
-Dfs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
|
||||
-libjars ./seaweedfs-hadoop2-client-1.6.6.jar \
|
||||
-libjars ./seaweedfs-hadoop2-client-1.6.7.jar \
|
||||
-ls /
|
||||
# or on hadoop3
|
||||
$ bin/hdfs dfs -Dfs.defaultFS=seaweedfs://localhost:8888 \
|
||||
-Dfs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem \
|
||||
-libjars ./seaweedfs-hadoop3-client-1.6.6.jar \
|
||||
-libjars ./seaweedfs-hadoop3-client-1.6.7.jar \
|
||||
-ls /
|
||||
|
||||
```
|
||||
|
|
@ -112,9 +112,9 @@ $ bin/hadoop classpath
|
|||
# Copy SeaweedFS HDFS client jar to one of the folders
|
||||
$ cd ${HADOOP_HOME}
|
||||
# for hadoop2
|
||||
$ cp ./seaweedfs-hadoop2-client-1.6.6.jar share/hadoop/common/lib/
|
||||
$ cp ./seaweedfs-hadoop2-client-1.6.7.jar share/hadoop/common/lib/
|
||||
# or for hadoop3
|
||||
$ cp ./seaweedfs-hadoop3-client-1.6.6.jar share/hadoop/common/lib/
|
||||
$ cp ./seaweedfs-hadoop3-client-1.6.7.jar share/hadoop/common/lib/
|
||||
```
|
||||
|
||||
Now you can do this:
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ The installation steps are divided into 2 steps:
|
|||
* https://cwiki.apache.org/confluence/display/Hive/AdminManual+Metastore+Administration
|
||||
|
||||
### Configure Hive Metastore to support SeaweedFS
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.6.jar to hive lib directory,for example:
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.7.jar to hive lib directory,for example:
|
||||
```
|
||||
cp seaweedfs-hadoop2-client-1.6.6.jar /opt/hadoop/share/hadoop/common/lib/
|
||||
cp seaweedfs-hadoop2-client-1.6.6.jar /opt/hive-metastore/lib/
|
||||
cp seaweedfs-hadoop2-client-1.6.7.jar /opt/hadoop/share/hadoop/common/lib/
|
||||
cp seaweedfs-hadoop2-client-1.6.7.jar /opt/hive-metastore/lib/
|
||||
```
|
||||
2. Modify core-site.xml
|
||||
modify core-site.xml to support SeaweedFS, 30888 is the filer port
|
||||
|
|
@ -50,9 +50,9 @@ metastore.thrift.port is the access port exposed by the Hive Metadata service it
|
|||
Follow instructions for installation of Presto:
|
||||
* https://prestosql.io/docs/current/installation/deployment.html
|
||||
### Configure Presto to support SeaweedFS
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.6.jar to Presto directory,for example:
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.7.jar to Presto directory,for example:
|
||||
```
|
||||
cp seaweedfs-hadoop2-client-1.6.6.jar /opt/presto-server-347/plugin/hive-hadoop2/
|
||||
cp seaweedfs-hadoop2-client-1.6.7.jar /opt/presto-server-347/plugin/hive-hadoop2/
|
||||
```
|
||||
2. Modify core-site.xml
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Installation for HBase
|
||||
|
||||
Two steps to run HBase on SeaweedFS
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.6.jar to `${HBASE_HOME}/lib`
|
||||
1. Copy the seaweedfs-hadoop2-client-1.6.7.jar to `${HBASE_HOME}/lib`
|
||||
1. And add the following 2 properties in `${HBASE_HOME}/conf/hbase-site.xml`
|
||||
|
||||
```
|
||||
|
|
|
|||
|
|
@ -11,12 +11,12 @@ To make these files visible to Spark, set HADOOP_CONF_DIR in $SPARK_HOME/conf/sp
|
|||
|
||||
## installation not inheriting from Hadoop cluster configuration
|
||||
|
||||
Copy the seaweedfs-hadoop2-client-1.6.6.jar to all executor machines.
|
||||
Copy the seaweedfs-hadoop2-client-1.6.7.jar to all executor machines.
|
||||
|
||||
Add the following to spark/conf/spark-defaults.conf on every node running Spark
|
||||
```
|
||||
spark.driver.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.executor.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.driver.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
spark.executor.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
```
|
||||
|
||||
And modify the configuration at runtime:
|
||||
|
|
@ -37,8 +37,8 @@ And modify the configuration at runtime:
|
|||
1. change the spark-defaults.conf
|
||||
|
||||
```
|
||||
spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem
|
||||
```
|
||||
|
||||
|
|
@ -81,8 +81,8 @@ spark.history.fs.cleaner.enabled=true
|
|||
spark.history.fs.logDirectory=seaweedfs://localhost:8888/spark2-history/
|
||||
spark.eventLog.dir=seaweedfs://localhost:8888/spark2-history/
|
||||
|
||||
spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.6.jar
|
||||
spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.6.7.jar
|
||||
spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem
|
||||
spark.hadoop.fs.defaultFS=seaweedfs://localhost:8888
|
||||
```
|
||||
|
|
|
|||
Loading…
Reference in a new issue