From c7b252fe79f1fd5c5ec0b726aadb282841a97594 Mon Sep 17 00:00:00 2001
From: Newnius <newnius.cn@gmail.com>
Date: Wed, 8 Aug 2018 12:12:12 +0800
Subject: [PATCH] update spark

---
 spark/2.2.1/README.md             |  2 +-
 spark/2.3.1/README.md             |  2 +-
 spark/test/config/core-site.xml   | 28 ++++++++++++++++++
 spark/test/config/hdfs-site.xml   | 40 +++++++++++++++++++++++++
 spark/test/config/mapred-site.xml | 32 ++++++++++++++++++++
 spark/test/config/slaves          |  3 ++
 spark/test/config/yarn-site.xml   | 49 +++++++++++++++++++++++++++++++
 7 files changed, 154 insertions(+), 2 deletions(-)
 create mode 100644 spark/test/config/core-site.xml
 create mode 100644 spark/test/config/hdfs-site.xml
 create mode 100644 spark/test/config/mapred-site.xml
 create mode 100644 spark/test/config/slaves
 create mode 100644 spark/test/config/yarn-site.xml

diff --git a/spark/2.2.1/README.md b/spark/2.2.1/README.md
index 0a381f7..b6a6152 100644
--- a/spark/2.2.1/README.md
+++ b/spark/2.2.1/README.md
@@ -34,7 +34,7 @@ spark-submit \
 	--master spark://spark-master:7077 \
 	--deploy-mode cluster \
 	--class org.apache.spark.examples.JavaSparkPi \
-	./examples/jars/spark-examples_2.11-2.2.1.jar 100
+	./examples/jars/spark-examples*.jar 100
 ```
 
 #### spark-shell HDFS wordcount
diff --git a/spark/2.3.1/README.md b/spark/2.3.1/README.md
index bd4be30..6a76449 100644
--- a/spark/2.3.1/README.md
+++ b/spark/2.3.1/README.md
@@ -34,7 +34,7 @@ spark-submit \
 	--master spark://spark-master:7077 \
 	--deploy-mode cluster \
 	--class org.apache.spark.examples.JavaSparkPi \
-	./examples/jars/spark-examples_2.11-2.3.1.jar 100
+	./examples/jars/spark-examples*.jar 100
 ```
 
 #### spark-shell HDFS wordcount
diff --git a/spark/test/config/core-site.xml b/spark/test/config/core-site.xml
new file mode 100644
index 0000000..7bd76e9
--- /dev/null
+++ b/spark/test/config/core-site.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>		 
+		<name>fs.defaultFS</name>
+		<value>hdfs://hadoop-master:8020</value>
+	</property>
+	<property>
+		<name>fs.default.name</name>
+		<value>hdfs://hadoop-master:8020</value>
+	</property>
+</configuration>
diff --git a/spark/test/config/hdfs-site.xml b/spark/test/config/hdfs-site.xml
new file mode 100644
index 0000000..ba3613a
--- /dev/null
+++ b/spark/test/config/hdfs-site.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>
+		<name>dfs.permissions</name>
+		<value>false</value>
+	</property>
+	<property>
+		<name>dfs.namenode.secondary.http-address</name>
+		<value>hadoop-slave1:50090</value>
+	</property>
+	<property>
+		<name>dfs.namenode.http-address</name>
+		<value>hadoop-master:50070</value>
+	</property>
+	<property>
+		<name>dfs.datanode.max.transfer.threads</name>
+		<value>8192</value>
+	</property>
+	<property>
+		<name>dfs.replication</name>
+		<value>3</value>
+	</property>
+</configuration>
diff --git a/spark/test/config/mapred-site.xml b/spark/test/config/mapred-site.xml
new file mode 100644
index 0000000..00fc7b8
--- /dev/null
+++ b/spark/test/config/mapred-site.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>
+		<name>mapreduce.framework.name</name>
+		<value>yarn</value>
+	</property>
+	<property>
+		<name>mapreduce.jobhistory.address</name>
+		<value>hadoop-master:10020</value>
+	</property>
+	<property>
+		<name>mapreduce.jobhistory.webapp.address</name>
+		<value>hadoop-master:19888</value>
+	</property>
+</configuration>
diff --git a/spark/test/config/slaves b/spark/test/config/slaves
new file mode 100644
index 0000000..3bb91be
--- /dev/null
+++ b/spark/test/config/slaves
@@ -0,0 +1,3 @@
+hadoop-slave1
+hadoop-slave2
+hadoop-slave3
diff --git a/spark/test/config/yarn-site.xml b/spark/test/config/yarn-site.xml
new file mode 100644
index 0000000..c3fec7b
--- /dev/null
+++ b/spark/test/config/yarn-site.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- Site specific YARN configuration properties -->
+<configuration>
+	<property>
+		<name>yarn.application.classpath</name>
+		<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
+	</property>
+	<property>
+		<name>yarn.resourcemanager.hostname</name>
+		<value>hadoop-master</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.aux-services</name>
+		<value>mapreduce_shuffle</value>
+	</property>
+	<property>
+		<name>yarn.log-aggregation-enable</name>
+		<value>true</value>
+	</property>
+	<property>
+		<name>yarn.log-aggregation.retain-seconds</name>
+		<value>604800</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.resource.memory-mb</name>
+		<value>2048</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.resource.cpu-vcores</name>
+		<value>2</value>
+	</property>
+	<property>
+		<name>yarn.scheduler.minimum-allocation-mb</name>
+		<value>1024</value>
+	</property>	
+</configuration>