diff --git a/hadoop/2.7.1/Dockerfile b/hadoop/2.7.1/Dockerfile index 52896a6..8d88ec0 100644 --- a/hadoop/2.7.1/Dockerfile +++ b/hadoop/2.7.1/Dockerfile @@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop # Default Conf Files -ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml -ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml -ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml -ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml -ADD slaves $HADOOP_HOME/etc/hadoop/slaves +ADD config/* $HADOOP_HOME/etc/hadoop/ RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh @@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME ADD bootstrap.sh /etc/bootstrap.sh -CMD ["/etc/bootstrap.sh", "-d"] +CMD ["/etc/bootstrap.sh"] diff --git a/hadoop/2.7.1/README.md b/hadoop/2.7.1/README.md index 1ce6a2d..f9351a6 100644 --- a/hadoop/2.7.1/README.md +++ b/hadoop/2.7.1/README.md @@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input bin/hadoop dfs -put etc/hadoop/* /user/root/input # submit the job -bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+' +bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+' ``` ## Browse the web UI diff --git a/hadoop/2.7.1/bootstrap.sh b/hadoop/2.7.1/bootstrap.sh index 4289858..67dc148 100755 --- a/hadoop/2.7.1/bootstrap.sh +++ b/hadoop/2.7.1/bootstrap.sh @@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh $HADOOP_PREFIX/sbin/start-yarn.sh $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver -if [[ $1 == "-d" ]]; then - while true; do sleep 1000; done -fi - -if [[ $1 == "-bash" ]]; then - /bin/bash -fi +while true; do sleep 1000; done diff --git a/hadoop/2.7.1/core-site.xml b/hadoop/2.7.1/config/core-site.xml similarity index 100% rename from hadoop/2.7.1/core-site.xml rename to hadoop/2.7.1/config/core-site.xml diff --git a/hadoop/2.7.1/hdfs-site.xml b/hadoop/2.7.1/config/hdfs-site.xml similarity index 100% rename from hadoop/2.7.1/hdfs-site.xml rename to hadoop/2.7.1/config/hdfs-site.xml diff --git a/hadoop/2.7.1/mapred-site.xml b/hadoop/2.7.1/config/mapred-site.xml similarity index 100% rename from hadoop/2.7.1/mapred-site.xml rename to hadoop/2.7.1/config/mapred-site.xml diff --git a/hadoop/2.7.1/slaves b/hadoop/2.7.1/config/slaves similarity index 100% rename from hadoop/2.7.1/slaves rename to hadoop/2.7.1/config/slaves diff --git a/hadoop/2.7.1/yarn-site.xml b/hadoop/2.7.1/config/yarn-site.xml similarity index 95% rename from hadoop/2.7.1/yarn-site.xml rename to hadoop/2.7.1/config/yarn-site.xml index c3fec7b..0759ee1 100644 --- a/hadoop/2.7.1/yarn-site.xml +++ b/hadoop/2.7.1/config/yarn-site.xml @@ -46,4 +46,8 @@ yarn.scheduler.minimum-allocation-mb 1024 + + yarn.nodemanager.vmem-pmem-ratio + 5 + diff --git a/hadoop/2.7.4/Dockerfile b/hadoop/2.7.4/Dockerfile index 211264d..8cdd314 100644 --- a/hadoop/2.7.4/Dockerfile +++ b/hadoop/2.7.4/Dockerfile @@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop # Default Conf Files -ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml -ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml -ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml -ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml -ADD slaves $HADOOP_HOME/etc/hadoop/slaves +ADD config/* $HADOOP_HOME/etc/hadoop/ RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh @@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME ADD bootstrap.sh /etc/bootstrap.sh -CMD ["/etc/bootstrap.sh", "-d"] +CMD ["/etc/bootstrap.sh"] diff --git a/hadoop/2.7.4/README.md b/hadoop/2.7.4/README.md index a3c6e03..ed1d198 100644 --- a/hadoop/2.7.4/README.md +++ b/hadoop/2.7.4/README.md @@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input bin/hadoop dfs -put etc/hadoop/* /user/root/input # submit the job -bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+' +bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+' ``` ## Browse the web UI diff --git a/hadoop/2.7.4/bootstrap.sh b/hadoop/2.7.4/bootstrap.sh index 4289858..67dc148 100755 --- a/hadoop/2.7.4/bootstrap.sh +++ b/hadoop/2.7.4/bootstrap.sh @@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh $HADOOP_PREFIX/sbin/start-yarn.sh $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver -if [[ $1 == "-d" ]]; then - while true; do sleep 1000; done -fi - -if [[ $1 == "-bash" ]]; then - /bin/bash -fi +while true; do sleep 1000; done diff --git a/hadoop/2.7.4/core-site.xml b/hadoop/2.7.4/config/core-site.xml similarity index 100% rename from hadoop/2.7.4/core-site.xml rename to hadoop/2.7.4/config/core-site.xml diff --git a/hadoop/2.7.4/hdfs-site.xml b/hadoop/2.7.4/config/hdfs-site.xml similarity index 100% rename from hadoop/2.7.4/hdfs-site.xml rename to hadoop/2.7.4/config/hdfs-site.xml diff --git a/hadoop/2.7.4/mapred-site.xml b/hadoop/2.7.4/config/mapred-site.xml similarity index 100% rename from hadoop/2.7.4/mapred-site.xml rename to hadoop/2.7.4/config/mapred-site.xml diff --git a/hadoop/2.7.4/slaves b/hadoop/2.7.4/config/slaves similarity index 100% rename from hadoop/2.7.4/slaves rename to hadoop/2.7.4/config/slaves diff --git a/hadoop/2.7.4/yarn-site.xml b/hadoop/2.7.4/config/yarn-site.xml similarity index 95% rename from hadoop/2.7.4/yarn-site.xml rename to hadoop/2.7.4/config/yarn-site.xml index c3fec7b..0759ee1 100644 --- a/hadoop/2.7.4/yarn-site.xml +++ b/hadoop/2.7.4/config/yarn-site.xml @@ -46,4 +46,8 @@ yarn.scheduler.minimum-allocation-mb 1024 + + yarn.nodemanager.vmem-pmem-ratio + 5 + diff --git a/hadoop/2.8.1/Dockerfile b/hadoop/2.8.1/Dockerfile index 5af05ab..f2aa7ff 100644 --- a/hadoop/2.8.1/Dockerfile +++ b/hadoop/2.8.1/Dockerfile @@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop # Default Conf Files -ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml -ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml -ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml -ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml -ADD slaves $HADOOP_HOME/etc/hadoop/slaves +ADD config/* $HADOOP_HOME/etc/hadoop/ RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh @@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME ADD bootstrap.sh /etc/bootstrap.sh -CMD ["/etc/bootstrap.sh", "-d"] +CMD ["/etc/bootstrap.sh"] diff --git a/hadoop/2.8.1/README.md b/hadoop/2.8.1/README.md index 699eb03..1146ffd 100644 --- a/hadoop/2.8.1/README.md +++ b/hadoop/2.8.1/README.md @@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input bin/hadoop dfs -put etc/hadoop/* /user/root/input # submit the job -bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+' +bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+' ``` ## Browse the web UI diff --git a/hadoop/2.8.1/bootstrap.sh b/hadoop/2.8.1/bootstrap.sh index 4289858..67dc148 100755 --- a/hadoop/2.8.1/bootstrap.sh +++ b/hadoop/2.8.1/bootstrap.sh @@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh $HADOOP_PREFIX/sbin/start-yarn.sh $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver -if [[ $1 == "-d" ]]; then - while true; do sleep 1000; done -fi - -if [[ $1 == "-bash" ]]; then - /bin/bash -fi +while true; do sleep 1000; done diff --git a/hadoop/2.8.1/core-site.xml b/hadoop/2.8.1/config/core-site.xml similarity index 100% rename from hadoop/2.8.1/core-site.xml rename to hadoop/2.8.1/config/core-site.xml diff --git a/hadoop/2.8.1/hdfs-site.xml b/hadoop/2.8.1/config/hdfs-site.xml similarity index 100% rename from hadoop/2.8.1/hdfs-site.xml rename to hadoop/2.8.1/config/hdfs-site.xml diff --git a/hadoop/2.8.1/mapred-site.xml b/hadoop/2.8.1/config/mapred-site.xml similarity index 100% rename from hadoop/2.8.1/mapred-site.xml rename to hadoop/2.8.1/config/mapred-site.xml diff --git a/hadoop/2.8.1/slaves b/hadoop/2.8.1/config/slaves similarity index 100% rename from hadoop/2.8.1/slaves rename to hadoop/2.8.1/config/slaves diff --git a/hadoop/2.8.1/yarn-site.xml b/hadoop/2.8.1/config/yarn-site.xml similarity index 95% rename from hadoop/2.8.1/yarn-site.xml rename to hadoop/2.8.1/config/yarn-site.xml index c3fec7b..0759ee1 100644 --- a/hadoop/2.8.1/yarn-site.xml +++ b/hadoop/2.8.1/config/yarn-site.xml @@ -46,4 +46,8 @@ yarn.scheduler.minimum-allocation-mb 1024 + + yarn.nodemanager.vmem-pmem-ratio + 5 + diff --git a/hadoop/2.8.4/Dockerfile b/hadoop/2.8.4/Dockerfile index cab1c11..1ac9592 100644 --- a/hadoop/2.8.4/Dockerfile +++ b/hadoop/2.8.4/Dockerfile @@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop # Default Conf Files -ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml -ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml -ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml -ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml -ADD slaves $HADOOP_HOME/etc/hadoop/slaves +ADD config/* $HADOOP_HOME/etc/hadoop/ RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh @@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME ADD bootstrap.sh /etc/bootstrap.sh -CMD ["/etc/bootstrap.sh", "-d"] +CMD ["/etc/bootstrap.sh"] diff --git a/hadoop/2.8.4/README.md b/hadoop/2.8.4/README.md index 5292952..a8dc723 100644 --- a/hadoop/2.8.4/README.md +++ b/hadoop/2.8.4/README.md @@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input bin/hadoop dfs -put etc/hadoop/* /user/root/input # submit the job -bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+' +bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+' ``` ## Browse the web UI diff --git a/hadoop/2.8.4/bootstrap.sh b/hadoop/2.8.4/bootstrap.sh index 4289858..67dc148 100755 --- a/hadoop/2.8.4/bootstrap.sh +++ b/hadoop/2.8.4/bootstrap.sh @@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh $HADOOP_PREFIX/sbin/start-yarn.sh $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver -if [[ $1 == "-d" ]]; then - while true; do sleep 1000; done -fi - -if [[ $1 == "-bash" ]]; then - /bin/bash -fi +while true; do sleep 1000; done diff --git a/hadoop/2.8.4/core-site.xml b/hadoop/2.8.4/config/core-site.xml similarity index 100% rename from hadoop/2.8.4/core-site.xml rename to hadoop/2.8.4/config/core-site.xml diff --git a/hadoop/2.8.4/hdfs-site.xml b/hadoop/2.8.4/config/hdfs-site.xml similarity index 100% rename from hadoop/2.8.4/hdfs-site.xml rename to hadoop/2.8.4/config/hdfs-site.xml diff --git a/hadoop/2.8.4/mapred-site.xml b/hadoop/2.8.4/config/mapred-site.xml similarity index 100% rename from hadoop/2.8.4/mapred-site.xml rename to hadoop/2.8.4/config/mapred-site.xml diff --git a/hadoop/2.8.4/slaves b/hadoop/2.8.4/config/slaves similarity index 100% rename from hadoop/2.8.4/slaves rename to hadoop/2.8.4/config/slaves diff --git a/hadoop/2.8.4/yarn-site.xml b/hadoop/2.8.4/config/yarn-site.xml similarity index 95% rename from hadoop/2.8.4/yarn-site.xml rename to hadoop/2.8.4/config/yarn-site.xml index c3fec7b..0759ee1 100644 --- a/hadoop/2.8.4/yarn-site.xml +++ b/hadoop/2.8.4/config/yarn-site.xml @@ -46,4 +46,8 @@ yarn.scheduler.minimum-allocation-mb 1024 + + yarn.nodemanager.vmem-pmem-ratio + 5 + diff --git a/hadoop/2.9.1/Dockerfile b/hadoop/2.9.1/Dockerfile index b6341b7..c1ab0f6 100644 --- a/hadoop/2.9.1/Dockerfile +++ b/hadoop/2.9.1/Dockerfile @@ -42,11 +42,7 @@ ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop # Default Conf Files -ADD core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml -ADD hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml -ADD mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml -ADD yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml -ADD slaves $HADOOP_HOME/etc/hadoop/slaves +ADD config/* $HADOOP_HOME/etc/hadoop/ RUN sed -i "/^export JAVA_HOME/ s:.*:export JAVA_HOME=${JAVA_HOME}\nexport HADOOP_HOME=${HADOOP_HOME}\nexport HADOOP_PREFIX=${HADOOP_PREFIX}:" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh @@ -56,4 +52,4 @@ WORKDIR $HADOOP_HOME ADD bootstrap.sh /etc/bootstrap.sh -CMD ["/etc/bootstrap.sh", "-d"] +CMD ["/etc/bootstrap.sh"] diff --git a/hadoop/2.9.1/README.md b/hadoop/2.9.1/README.md index 99e3861..fe7b131 100644 --- a/hadoop/2.9.1/README.md +++ b/hadoop/2.9.1/README.md @@ -79,7 +79,7 @@ bin/hadoop dfs -mkdir -p /user/root/input bin/hadoop dfs -put etc/hadoop/* /user/root/input # submit the job -bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+' +bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar grep input output 'dfs[a-z.]+' ``` ## Browse the web UI diff --git a/hadoop/2.9.1/bootstrap.sh b/hadoop/2.9.1/bootstrap.sh index 4289858..67dc148 100755 --- a/hadoop/2.9.1/bootstrap.sh +++ b/hadoop/2.9.1/bootstrap.sh @@ -26,10 +26,4 @@ $HADOOP_PREFIX/sbin/start-dfs.sh $HADOOP_PREFIX/sbin/start-yarn.sh $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver -if [[ $1 == "-d" ]]; then - while true; do sleep 1000; done -fi - -if [[ $1 == "-bash" ]]; then - /bin/bash -fi +while true; do sleep 1000; done diff --git a/hadoop/2.9.1/core-site.xml b/hadoop/2.9.1/config/core-site.xml similarity index 100% rename from hadoop/2.9.1/core-site.xml rename to hadoop/2.9.1/config/core-site.xml diff --git a/hadoop/2.9.1/hdfs-site.xml b/hadoop/2.9.1/config/hdfs-site.xml similarity index 100% rename from hadoop/2.9.1/hdfs-site.xml rename to hadoop/2.9.1/config/hdfs-site.xml diff --git a/hadoop/2.9.1/mapred-site.xml b/hadoop/2.9.1/config/mapred-site.xml similarity index 100% rename from hadoop/2.9.1/mapred-site.xml rename to hadoop/2.9.1/config/mapred-site.xml diff --git a/hadoop/2.9.1/slaves b/hadoop/2.9.1/config/slaves similarity index 100% rename from hadoop/2.9.1/slaves rename to hadoop/2.9.1/config/slaves diff --git a/hadoop/2.9.1/config/yarn-site.xml b/hadoop/2.9.1/config/yarn-site.xml new file mode 100644 index 0000000..0759ee1 --- /dev/null +++ b/hadoop/2.9.1/config/yarn-site.xml @@ -0,0 +1,53 @@ + + + + + + yarn.application.classpath + /usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/* + + + yarn.resourcemanager.hostname + hadoop-master + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.log-aggregation-enable + true + + + yarn.log-aggregation.retain-seconds + 604800 + + + yarn.nodemanager.resource.memory-mb + 2048 + + + yarn.nodemanager.resource.cpu-vcores + 2 + + + yarn.scheduler.minimum-allocation-mb + 1024 + + + yarn.nodemanager.vmem-pmem-ratio + 5 + + diff --git a/hadoop/2.9.1/yarn-site.xml b/hadoop/2.9.1/yarn-site.xml deleted file mode 100644 index c3fec7b..0000000 --- a/hadoop/2.9.1/yarn-site.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - yarn.application.classpath - /usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/* - - - yarn.resourcemanager.hostname - hadoop-master - - - yarn.nodemanager.aux-services - mapreduce_shuffle - - - yarn.log-aggregation-enable - true - - - yarn.log-aggregation.retain-seconds - 604800 - - - yarn.nodemanager.resource.memory-mb - 2048 - - - yarn.nodemanager.resource.cpu-vcores - 2 - - - yarn.scheduler.minimum-allocation-mb - 1024 - -