YAO/YAO-docs
1
0
Fork 0
mirror of https://github.com/newnius/YAO-docs.git synced 2024-07-06 05:15:18 +00:00
Code Issues Releases Wiki Activity

update doc

Browse Source
This commit is contained in:
Newnius 2020-07-08 21:26:46 +08:00
parent 429e85b90c
commit 436f11c55e
8 changed files with 43 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sbin/run_agent.sh
View File

@ -18,7 +18,7 @@ docker run \
--env ClientExtHost=${ip} \ --env ClientExtHost=${ip} \
--env Port=8000 \ --env Port=8000 \
--env HeartbeatInterval=5 \ --env HeartbeatInterval=5 \
--env KafkaBrokers=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \ --env ReportAddress='http://yao-scheduler:8080/?action=agent_report' \
--env EnableEventTrigger='true' \ --env EnableEventTrigger='true' \
--env PYTHONUNBUFFERED=1 \ --env PYTHONUNBUFFERED=1 \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \

23
sbin/run_optimizer.sh
View File

@ -1,13 +1,16 @@
#!/bin/bash #!/bin/bash
docker run \ docker run \
--gpus all \ --name yao-optimizer \
--name yao-optimizer \ --hostname yao-optimizer \
--hostname yao-optimizer \ --network yao-net \
--network yao-net \ --network-alias yao-optimizer \
--network-alias yao-optimizer \ -d \
-d \ --restart always \
--restart always \ --detach=true \
--detach=true \ --env PYTHONUNBUFFERED=1 \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \
quickdeploy/yao-optimizer:dev quickdeploy/yao-optimizer:dev
#--gpus all \
#--gpus '"device=1"' \

5
sbin/start_agent_master.sh
View File

@ -2,9 +2,10 @@
docker service create \ docker service create \
--name yao-agent-master \ --name yao-agent-master \
--network yao-net \ --hostname yao-agent-master \
--constraint node.role==manager \ --constraint node.role==manager \
--mode global \ --network yao-net \
--replicas 1 \
--detach=true \ --detach=true \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \
--mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \

2
sbin/start_mysql.sh
View File

@ -11,5 +11,5 @@ docker service create \
-e MYSQL_DATABASE=yao \ -e MYSQL_DATABASE=yao \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \
mysql:5.7 mysql:5.7
#--mount type=bind,source=/data/mysql,target=/var/lib/mysql \ #--mount type=bind,source=/data/mysql,target=/var/lib/mysql \

2
sbin/start_portal.sh
View File

@ -9,7 +9,7 @@ docker service create \
--detach=true \ --detach=true \
--publish 80:80 \ --publish 80:80 \
--env SITE_SCHEDULER_ADDR=http://yao-scheduler:8080 \ --env SITE_SCHEDULER_ADDR=http://yao-scheduler:8080 \
--env SITE_BASE_URL=http://210.28.132.13 \ --env SITE_BASE_URL=http://yao.pasalab.jluapp.com \
--env MYSQL_HOST=mysql \ --env MYSQL_HOST=mysql \
--env MYSQL_PORT=3306 \ --env MYSQL_PORT=3306 \
--env MYSQL_DATABASE=yao \ --env MYSQL_DATABASE=yao \

1
sbin/start_redis.sh
View File

@ -7,4 +7,5 @@ docker service create \
--replicas 1 \ --replicas 1 \
--detach=true \ --detach=true \
--endpoint-mode dnsrr \ --endpoint-mode dnsrr \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \
redis redis-server --appendonly yes redis redis-server --appendonly yes

18
sbin/start_scheduler.sh
View File

@ -7,5 +7,21 @@ docker service create \
--network yao-net \ --network yao-net \
--replicas 1 \ --replicas 1 \
--detach=true \ --detach=true \
--env KafkaBrokers=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \
--env KafkaTopic=yao \
--env SchedulerPolicy=fair \
--env ListenAddr='0.0.0.0:8080' \
--env HDFSAddress='' \
--env HDFSBaseDir='/user/yao/output/' \
--env DFSBaseDir='/dfs/yao-jobs/' \
--env EnableShareRatio=1.75 \
--env ShareMaxUtilization=1.30 \
--env EnablePreScheduleRatio=1.75 \
--env PreScheduleExtraTime=15 \
--env PreScheduleTimeout=300 \
--mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \
quickdeploy/yao-scheduler:dev sleep infinity quickdeploy/yao-scheduler:dev
#--env HDFSAddress='http://192.168.100.104:50070/' \
#--env LoggerOutputDir='log/' \
#quickdeploy/yao-scheduler:dev sleep infinity

33
setup.md
View File

@ -27,43 +27,22 @@ docker swarm join --token A-LONG-TOKEN-STRING-HERE 192.168.0.1:2377
## Create an overlay network named `yao` ## Create an overlay network named `yao`
```bash ```bash
docker network create --driver overlay --attachable --opt encrypted yao-net docker network create --driver overlay --attachable yao-net
# docker network create --driver overlay --attachable --opt encrypted yao-net
``` ```
*Note: try remove encrypted when the containers cannot communicate cross nodes* *Note: try remove encrypted when the containers cannot communicate cross nodes*
## Start zk cluster ## Start HDFS cluster (Optional)
```bash
sbin/start_zks.sh
```
## Start HDFS cluster
```bash ```bash
sbin/run_hdfs.sh sbin/run_hdfs.sh
``` ```
## Start Kafka cluster ## Start GlusterFS cluster (Optional)
```bash ```bash
sbin/start_kafka.sh sbin/start_glusterfs.sh
```
## Create kafka topic yao
```bash
# delete old
bin/kafka-topics.sh \
--delete \
--zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \
--topic yao
# create new
bin/kafka-topics.sh \
--create \
--zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \
--replication-factor 3 \
--partitions 1 \
--topic yao
``` ```