From a3765355455ff6a11dcb5021a90ea9979ca7c464 Mon Sep 17 00:00:00 2001
From: Newnius <newnius.cn@gmail.com>
Date: Wed, 8 Aug 2018 15:51:43 +0800
Subject: [PATCH] update spark

---
 spark/2.2.1-yarn/README.md            | 20 ++++++++++++++++++++
 spark/2.2.1-yarn/config/yarn-site.xml |  4 ++++
 spark/2.3.1-yarn/README.md            | 20 ++++++++++++++++++++
 spark/2.3.1-yarn/config/yarn-site.xml |  4 ++++
 4 files changed, 48 insertions(+)

diff --git a/spark/2.2.1-yarn/README.md b/spark/2.2.1-yarn/README.md
index b207c1f..1b3c118 100644
--- a/spark/2.2.1-yarn/README.md
+++ b/spark/2.2.1-yarn/README.md
@@ -9,6 +9,7 @@ docker service create \
 	--network swarm-net \
 	--replicas 1 \
 	--detach true \
+  --mount type=bind,source=/etc/localtime,target=/etc/localtime \
 	newnius/spark:2.2.1-yarn
 ```
 
@@ -43,3 +44,22 @@ val cnt = words.map(word => 1).reduce(_ + _)
 ## Browse the web UI
 
 In Spark On Yarn mode, the spark jobs will occur in the Yarn web UI.
+
+## Custom configuration
+
+To persist data or modify the conf files, refer to the following script.
+
+The `/config/hadoop` path is where new conf files to be replaces, you don't have to put all the files.
+
+
+```bash
+docker service create \
+	--name spark-client \
+	--hostname spark-client \
+	--network swarm-net \
+	--replicas 1 \
+	--detach true \
+  --mount type=bind,source=/etc/localtime,target=/etc/localtime \
+  --mount type=bind,source=/data/hadoop/config,target=/config/hadoop \
+	newnius/spark:2.2.1-yarn
+```
diff --git a/spark/2.2.1-yarn/config/yarn-site.xml b/spark/2.2.1-yarn/config/yarn-site.xml
index c3fec7b..0759ee1 100644
--- a/spark/2.2.1-yarn/config/yarn-site.xml
+++ b/spark/2.2.1-yarn/config/yarn-site.xml
@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>
diff --git a/spark/2.3.1-yarn/README.md b/spark/2.3.1-yarn/README.md
index 21c6037..9fc9eaf 100644
--- a/spark/2.3.1-yarn/README.md
+++ b/spark/2.3.1-yarn/README.md
@@ -9,6 +9,7 @@ docker service create \
 	--network swarm-net \
 	--replicas 1 \
 	--detach true \
+  --mount type=bind,source=/etc/localtime,target=/etc/localtime \
 	newnius/spark:2.3.1-yarn
 ```
 
@@ -43,3 +44,22 @@ val cnt = words.map(word => 1).reduce(_ + _)
 ## Browse the web UI
 
 In Spark On Yarn mode, the spark jobs will occur in the Yarn web UI.
+
+## Custom configuration
+
+To persist data or modify the conf files, refer to the following script.
+
+The `/config/hadoop` path is where new conf files to be replaces, you don't have to put all the files.
+
+
+```bash
+docker service create \
+	--name spark-client \
+	--hostname spark-client \
+	--network swarm-net \
+	--replicas 1 \
+	--detach true \
+  --mount type=bind,source=/etc/localtime,target=/etc/localtime \
+  --mount type=bind,source=/data/hadoop/config,target=/config/hadoop \
+	newnius/spark:2.3.1-yarn
+```
diff --git a/spark/2.3.1-yarn/config/yarn-site.xml b/spark/2.3.1-yarn/config/yarn-site.xml
index c3fec7b..0759ee1 100644
--- a/spark/2.3.1-yarn/config/yarn-site.xml
+++ b/spark/2.3.1-yarn/config/yarn-site.xml
@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>