From ae6ed170b5c3bc622653217d0807ac483f45c446 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 25 Oct 2020 14:46:03 -0700 Subject: [PATCH] 1.5.2 --- Hadoop-Benchmark.md | 2 +- Hadoop-Compatible-File-System.md | 8 ++++---- run-Spark-on-SeaweedFS.md | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Hadoop-Benchmark.md b/Hadoop-Benchmark.md index d7483ac..e984701 100644 --- a/Hadoop-Benchmark.md +++ b/Hadoop-Benchmark.md @@ -26,7 +26,7 @@ Then get the seaweedfs hadoop client jar. ``` cd share/hadoop/common/lib/ -wget https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.5.1/seaweedfs-hadoop2-client-1.5.1.jar +wget https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.5.2/seaweedfs-hadoop2-client-1.5.2.jar ``` # TestDFSIO Benchmark diff --git a/Hadoop-Compatible-File-System.md b/Hadoop-Compatible-File-System.md index 9d59f0c..8ad38d9 100644 --- a/Hadoop-Compatible-File-System.md +++ b/Hadoop-Compatible-File-System.md @@ -23,7 +23,7 @@ Maven com.github.chrislusf seaweedfs-hadoop3-client - 1.5.1 + 1.5.2 or @@ -31,16 +31,16 @@ or com.github.chrislusf seaweedfs-hadoop2-client - 1.5.1 + 1.5.2 ``` Or you can download the latest version from MavenCentral * https://mvnrepository.com/artifact/com.github.chrislusf/seaweedfs-hadoop2-client - * [seaweedfs-hadoop2-client-1.5.1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.5.1/seaweedfs-hadoop2-client-1.5.1.jar) + * [seaweedfs-hadoop2-client-1.5.2.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop2-client/1.5.2/seaweedfs-hadoop2-client-1.5.2.jar) * https://mvnrepository.com/artifact/com.github.chrislusf/seaweedfs-hadoop3-client - * [seaweedfs-hadoop3-client-1.5.1.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop3-client/1.5.1/seaweedfs-hadoop3-client-1.5.1.jar) + * [seaweedfs-hadoop3-client-1.5.2.jar](https://oss.sonatype.org/service/local/repositories/releases/content/com/github/chrislusf/seaweedfs-hadoop3-client/1.5.2/seaweedfs-hadoop3-client-1.5.2.jar) # Test SeaweedFS on Hadoop diff --git a/run-Spark-on-SeaweedFS.md b/run-Spark-on-SeaweedFS.md index e297798..6717192 100644 --- a/run-Spark-on-SeaweedFS.md +++ b/run-Spark-on-SeaweedFS.md @@ -11,12 +11,12 @@ To make these files visible to Spark, set HADOOP_CONF_DIR in $SPARK_HOME/conf/sp ## installation not inheriting from Hadoop cluster configuration -Copy the seaweedfs-hadoop2-client-1.5.1.jar to all executor machines. +Copy the seaweedfs-hadoop2-client-1.5.2.jar to all executor machines. Add the following to spark/conf/spark-defaults.conf on every node running Spark ``` -spark.driver.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.5.1.jar -spark.executor.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.5.1.jar +spark.driver.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.5.2.jar +spark.executor.extraClassPath=/path/to/seaweedfs-hadoop2-client-1.5.2.jar ``` And modify the configuration at runtime: @@ -37,8 +37,8 @@ And modify the configuration at runtime: 1. change the spark-defaults.conf ``` -spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.1.jar -spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.1.jar +spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.2.jar +spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.2.jar spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem ``` @@ -81,8 +81,8 @@ spark.history.fs.cleaner.enabled=true spark.history.fs.logDirectory=seaweedfs://localhost:8888/spark2-history/ spark.eventLog.dir=seaweedfs://localhost:8888/spark2-history/ -spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.1.jar -spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.1.jar +spark.driver.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.2.jar +spark.executor.extraClassPath=/Users/chris/go/src/github.com/chrislusf/seaweedfs/other/java/hdfs2/target/seaweedfs-hadoop2-client-1.5.2.jar spark.hadoop.fs.seaweedfs.impl=seaweed.hdfs.SeaweedFileSystem spark.hadoop.fs.defaultFS=seaweedfs://localhost:8888 ```