mirror of
https://github.com/newnius/Dockerfiles.git
synced 2025-12-13 09:36:44 +00:00
update hadoop
This commit is contained in:
@@ -4,41 +4,41 @@
|
|||||||
|
|
||||||
`--hostname` needs docker 1.13 or higher
|
`--hostname` needs docker 1.13 or higher
|
||||||
|
|
||||||
```
|
```bash
|
||||||
docker service create \
|
docker service create \
|
||||||
--name hadoop-master \
|
--name hadoop-master \
|
||||||
--network swarm-net \
|
|
||||||
--hostname hadoop-master \
|
--hostname hadoop-master \
|
||||||
|
--network swarm-net \
|
||||||
--replicas 1 \
|
--replicas 1 \
|
||||||
--endpoint-mode dnsrr \
|
--endpoint-mode dnsrr \
|
||||||
newnius/hadoop
|
newnius/hadoop
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```bash
|
||||||
docker service create \
|
docker service create \
|
||||||
--name hadoop-slave1 \
|
--name hadoop-slave1 \
|
||||||
--network swarm-net \
|
|
||||||
--hostname hadoop-slave1 \
|
--hostname hadoop-slave1 \
|
||||||
|
--network swarm-net \
|
||||||
--replicas 1 \
|
--replicas 1 \
|
||||||
--endpoint-mode dnsrr \
|
--endpoint-mode dnsrr \
|
||||||
newnius/hadoop
|
newnius/hadoop
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```bash
|
||||||
docker service create \
|
docker service create \
|
||||||
--name hadoop-slave2 \
|
--name hadoop-slave2 \
|
||||||
--network swarm-net \
|
|
||||||
--hostname hadoop-slave2 \
|
--hostname hadoop-slave2 \
|
||||||
|
--network swarm-net \
|
||||||
--replicas 1 \
|
--replicas 1 \
|
||||||
--endpoint-mode dnsrr \
|
--endpoint-mode dnsrr \
|
||||||
newnius/hadoop
|
newnius/hadoop
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```bash
|
||||||
docker service create \
|
docker service create \
|
||||||
--name hadoop-slave3 \
|
--name hadoop-slave3 \
|
||||||
--network swarm-net \
|
|
||||||
--hostname hadoop-slave3 \
|
--hostname hadoop-slave3 \
|
||||||
|
--network swarm-net \
|
||||||
--replicas 1 \
|
--replicas 1 \
|
||||||
--endpoint-mode dnsrr \
|
--endpoint-mode dnsrr \
|
||||||
newnius/hadoop
|
newnius/hadoop
|
||||||
@@ -49,33 +49,31 @@ newnius/hadoop
|
|||||||
In the first deploy, format dfs first
|
In the first deploy, format dfs first
|
||||||
|
|
||||||
### stop cluster (in master)
|
### stop cluster (in master)
|
||||||
`sbin/stop-yarn.sh`
|
```bash
|
||||||
`sbin/stop-dfs.sh`
|
sbin/stop-dfs.sh
|
||||||
|
```
|
||||||
### remove previous data (in all nodes)
|
|
||||||
clear all data in /tmp in all nodes
|
|
||||||
|
|
||||||
### format hdfs (in master)
|
### format hdfs (in master)
|
||||||
```
|
```bash
|
||||||
bin/hadoop namenode -format
|
bin/hadoop namenode -format
|
||||||
```
|
```
|
||||||
|
|
||||||
### start cluster (in master)
|
### start cluster (in master)
|
||||||
`sbin/start-dfs.sh`
|
```bash
|
||||||
`sbin/start-yarn.sh`
|
sbin/start-dfs.sh
|
||||||
|
```
|
||||||
|
|
||||||
### Run a test job
|
### Run a test job
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# prepare input data
|
||||||
|
bin/hadoop dfs -mkdir -p /user/root/input
|
||||||
|
|
||||||
|
bin/hadoop dfs -put etc/hadoop/* /user/root/input
|
||||||
|
```
|
||||||
|
```bash
|
||||||
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
|
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.4.jar grep input output 'dfs[a-z.]+'
|
||||||
|
|
||||||
Note: there is no such dir default, you can add data by
|
|
||||||
|
|
||||||
```
|
|
||||||
bin/hadoop dfs -mkdir -p /user/root/input
|
|
||||||
```
|
|
||||||
and
|
|
||||||
```
|
|
||||||
bin/hadoop dfs -put etc/hadoop/* /user/root/input
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### monitor cluster in browser
|
### monitor cluster in browser
|
||||||
|
|||||||
Reference in New Issue
Block a user