update hadoop

hadoop/2.7.1/Dockerfile

@@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 
 # Default Conf Files
-ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/* $HADOOP_HOME/etc/hadoop/
-ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
-ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
-ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
-ADD slaves $HADOOP_HOME/etc/hadoop/slaves
 
 RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 
@@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME
 
 ADD bootstrap.sh /etc/bootstrap.sh
 
-CMD ["/etc/bootstrap.sh", "-d"]
+CMD ["/etc/bootstrap.sh"]

hadoop/2.7.1/README.md

@@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input
 bin/hadoop dfs -put etc/hadoop/* /user/root/input
 
 # submit the job
-bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
+bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+'
 ```
 
 ## Browse the web UI

hadoop/2.7.1/bootstrap.sh

@@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh
 $HADOOP_PREFIX/sbin/start-yarn.sh
 $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
 
-if [[ $1 == "-d" ]]; then
+while true; do sleep 1000; done
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi

hadoop/2.7.1/config/yarn-site.xml

@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>

hadoop/2.7.4/Dockerfile

@@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 
 # Default Conf Files
-ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/* $HADOOP_HOME/etc/hadoop/
-ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
-ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
-ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
-ADD slaves $HADOOP_HOME/etc/hadoop/slaves
 
 RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 
@@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME
 
 ADD bootstrap.sh /etc/bootstrap.sh
 
-CMD ["/etc/bootstrap.sh", "-d"]
+CMD ["/etc/bootstrap.sh"]

hadoop/2.7.4/README.md

@@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input
 bin/hadoop dfs -put etc/hadoop/* /user/root/input
 
 # submit the job
-bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
+bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+'
 ```
 
 ## Browse the web UI

hadoop/2.7.4/bootstrap.sh

@@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh
 $HADOOP_PREFIX/sbin/start-yarn.sh
 $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
 
-if [[ $1 == "-d" ]]; then
+while true; do sleep 1000; done
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi

hadoop/2.7.4/config/yarn-site.xml

@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>

hadoop/2.8.1/Dockerfile

@@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 
 # Default Conf Files
-ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/* $HADOOP_HOME/etc/hadoop/
-ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
-ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
-ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
-ADD slaves $HADOOP_HOME/etc/hadoop/slaves
 
 RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 
@@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME
 
 ADD bootstrap.sh /etc/bootstrap.sh
 
-CMD ["/etc/bootstrap.sh", "-d"]
+CMD ["/etc/bootstrap.sh"]

hadoop/2.8.1/README.md

@@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input
 bin/hadoop dfs -put etc/hadoop/* /user/root/input
 
 # submit the job
-bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
+bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+'
 ```
 
 ## Browse the web UI

hadoop/2.8.1/bootstrap.sh

@@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh
 $HADOOP_PREFIX/sbin/start-yarn.sh
 $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
 
-if [[ $1 == "-d" ]]; then
+while true; do sleep 1000; done
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi

hadoop/2.8.1/config/yarn-site.xml

@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>

hadoop/2.8.4/Dockerfile

@@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 
 # Default Conf Files
-ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/* $HADOOP_HOME/etc/hadoop/
-ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
-ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
-ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
-ADD slaves $HADOOP_HOME/etc/hadoop/slaves
 
 RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 
@@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME
 
 ADD bootstrap.sh /etc/bootstrap.sh
 
-CMD ["/etc/bootstrap.sh", "-d"]
+CMD ["/etc/bootstrap.sh"]

hadoop/2.8.4/README.md

@@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input
 bin/hadoop dfs -put etc/hadoop/* /user/root/input
 
 # submit the job
-bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
+bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+'
 ```
 
 ## Browse the web UI

hadoop/2.8.4/bootstrap.sh

@@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh
 $HADOOP_PREFIX/sbin/start-yarn.sh
 $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
 
-if [[ $1 == "-d" ]]; then
+while true; do sleep 1000; done
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi

hadoop/2.8.4/config/yarn-site.xml

@@ -46,4 +46,8 @@
 		<name>yarn.scheduler.minimum-allocation-mb</name>
 		<value>1024</value>
 	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
 </configuration>

hadoop/2.9.1/Dockerfile

@@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 
 # Default Conf Files
-ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml
+ADD config/* $HADOOP_HOME/etc/hadoop/
-ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
-ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
-ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
-ADD slaves $HADOOP_HOME/etc/hadoop/slaves
 
 RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
 
@@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME
 
 ADD bootstrap.sh /etc/bootstrap.sh
 
-CMD ["/etc/bootstrap.sh", "-d"]
+CMD ["/etc/bootstrap.sh"]

hadoop/2.9.1/README.md

@@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input
 bin/hadoop dfs -put etc/hadoop/* /user/root/input
 
 # submit the job
-bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
+bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+'
 ```
 
 ## Browse the web UI

hadoop/2.9.1/bootstrap.sh

@@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh
 $HADOOP_PREFIX/sbin/start-yarn.sh
 $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
 
-if [[ $1 == "-d" ]]; then
+while true; do sleep 1000; done
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi

hadoop/2.9.1/config/yarn-site.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- Site specific YARN configuration properties -->
+<configuration>
+	<property>
+		<name>yarn.application.classpath</name>
+		<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
+	</property>
+	<property>
+		<name>yarn.resourcemanager.hostname</name>
+		<value>hadoop-master</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.aux-services</name>
+		<value>mapreduce_shuffle</value>
+	</property>
+	<property>
+		<name>yarn.log-aggregation-enable</name>
+		<value>true</value>
+	</property>
+	<property>
+		<name>yarn.log-aggregation.retain-seconds</name>
+		<value>604800</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.resource.memory-mb</name>
+		<value>2048</value>
+	</property>
+	<property>
+		<name>yarn.nodemanager.resource.cpu-vcores</name>
+		<value>2</value>
+	</property>
+	<property>
+		<name>yarn.scheduler.minimum-allocation-mb</name>
+		<value>1024</value>
+	</property>	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>5</value>
+	</property>	
+</configuration>

hadoop/2.9.1/yarn-site.xml

@@ -1,49 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<!-- Site specific YARN configuration properties -->
-<configuration>
-	<property>
-		<name>yarn.application.classpath</name>
-		<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
-	</property>
-	<property>
-		<name>yarn.resourcemanager.hostname</name>
-		<value>hadoop-master</value>
-	</property>
-	<property>
-		<name>yarn.nodemanager.aux-services</name>
-		<value>mapreduce_shuffle</value>
-	</property>
-	<property>
-		<name>yarn.log-aggregation-enable</name>
-		<value>true</value>
-	</property>
-	<property>
-		<name>yarn.log-aggregation.retain-seconds</name>
-		<value>604800</value>
-	</property>
-	<property>
-		<name>yarn.nodemanager.resource.memory-mb</name>
-		<value>2048</value>
-	</property>
-	<property>
-		<name>yarn.nodemanager.resource.cpu-vcores</name>
-		<value>2</value>
-	</property>
-	<property>
-		<name>yarn.scheduler.minimum-allocation-mb</name>
-		<value>1024</value>
-	</property>	
-</configuration>