From 436f11c55ea96ed48b919e280838beff251bc839 Mon Sep 17 00:00:00 2001 From: Newnius Date: Wed, 8 Jul 2020 21:26:46 +0800 Subject: [PATCH] update doc --- sbin/run_agent.sh | 2 +- sbin/run_optimizer.sh | 23 +++++++++++++---------- sbin/start_agent_master.sh | 5 +++-- sbin/start_mysql.sh | 2 +- sbin/start_portal.sh | 2 +- sbin/start_redis.sh | 1 + sbin/start_scheduler.sh | 18 +++++++++++++++++- setup.md | 33 ++++++--------------------------- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/sbin/run_agent.sh b/sbin/run_agent.sh index 1d0c5f7..dbae85d 100755 --- a/sbin/run_agent.sh +++ b/sbin/run_agent.sh @@ -18,7 +18,7 @@ docker run \ --env ClientExtHost=${ip} \ --env Port=8000 \ --env HeartbeatInterval=5 \ - --env KafkaBrokers=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \ + --env ReportAddress='http://yao-scheduler:8080/?action=agent_report' \ --env EnableEventTrigger='true' \ --env PYTHONUNBUFFERED=1 \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ diff --git a/sbin/run_optimizer.sh b/sbin/run_optimizer.sh index 61969c8..5907679 100755 --- a/sbin/run_optimizer.sh +++ b/sbin/run_optimizer.sh @@ -1,13 +1,16 @@ #!/bin/bash docker run \ - --gpus all \ - --name yao-optimizer \ - --hostname yao-optimizer \ - --network yao-net \ - --network-alias yao-optimizer \ - -d \ - --restart always \ - --detach=true \ - --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ - quickdeploy/yao-optimizer:dev + --name yao-optimizer \ + --hostname yao-optimizer \ + --network yao-net \ + --network-alias yao-optimizer \ + -d \ + --restart always \ + --detach=true \ + --env PYTHONUNBUFFERED=1 \ + --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ + quickdeploy/yao-optimizer:dev + +#--gpus all \ +#--gpus '"device=1"' \ diff --git a/sbin/start_agent_master.sh b/sbin/start_agent_master.sh index cc2d352..211fc48 100755 --- a/sbin/start_agent_master.sh +++ b/sbin/start_agent_master.sh @@ -2,9 +2,10 @@ docker service create \ --name yao-agent-master \ - --network yao-net \ + --hostname yao-agent-master \ --constraint node.role==manager \ - --mode global \ + --network yao-net \ + --replicas 1 \ --detach=true \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock \ diff --git a/sbin/start_mysql.sh b/sbin/start_mysql.sh index da12795..b0db393 100755 --- a/sbin/start_mysql.sh +++ b/sbin/start_mysql.sh @@ -11,5 +11,5 @@ docker service create \ -e MYSQL_DATABASE=yao \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ mysql:5.7 - + #--mount type=bind,source=/data/mysql,target=/var/lib/mysql \ diff --git a/sbin/start_portal.sh b/sbin/start_portal.sh index be7e471..84d125e 100755 --- a/sbin/start_portal.sh +++ b/sbin/start_portal.sh @@ -9,7 +9,7 @@ docker service create \ --detach=true \ --publish 80:80 \ --env SITE_SCHEDULER_ADDR=http://yao-scheduler:8080 \ - --env SITE_BASE_URL=http://210.28.132.13 \ + --env SITE_BASE_URL=http://yao.pasalab.jluapp.com \ --env MYSQL_HOST=mysql \ --env MYSQL_PORT=3306 \ --env MYSQL_DATABASE=yao \ diff --git a/sbin/start_redis.sh b/sbin/start_redis.sh index d0f3103..225dbb5 100755 --- a/sbin/start_redis.sh +++ b/sbin/start_redis.sh @@ -7,4 +7,5 @@ docker service create \ --replicas 1 \ --detach=true \ --endpoint-mode dnsrr \ + --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ redis redis-server --appendonly yes diff --git a/sbin/start_scheduler.sh b/sbin/start_scheduler.sh index aef1e64..0b7edbe 100755 --- a/sbin/start_scheduler.sh +++ b/sbin/start_scheduler.sh @@ -7,5 +7,21 @@ docker service create \ --network yao-net \ --replicas 1 \ --detach=true \ + --env KafkaBrokers=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \ + --env KafkaTopic=yao \ + --env SchedulerPolicy=fair \ + --env ListenAddr='0.0.0.0:8080' \ + --env HDFSAddress='' \ + --env HDFSBaseDir='/user/yao/output/' \ + --env DFSBaseDir='/dfs/yao-jobs/' \ + --env EnableShareRatio=1.75 \ + --env ShareMaxUtilization=1.30 \ + --env EnablePreScheduleRatio=1.75 \ + --env PreScheduleExtraTime=15 \ + --env PreScheduleTimeout=300 \ --mount type=bind,source=/etc/localtime,target=/etc/localtime,readonly \ - quickdeploy/yao-scheduler:dev sleep infinity + quickdeploy/yao-scheduler:dev + + #--env HDFSAddress='http://192.168.100.104:50070/' \ + #--env LoggerOutputDir='log/' \ + #quickdeploy/yao-scheduler:dev sleep infinity diff --git a/setup.md b/setup.md index 16c80ee..d15d580 100644 --- a/setup.md +++ b/setup.md @@ -27,43 +27,22 @@ docker swarm join --token A-LONG-TOKEN-STRING-HERE 192.168.0.1:2377 ## Create an overlay network named `yao` ```bash -docker network create --driver overlay --attachable --opt encrypted yao-net +docker network create --driver overlay --attachable yao-net + +# docker network create --driver overlay --attachable --opt encrypted yao-net ``` *Note: try remove encrypted when the containers cannot communicate cross nodes* -## Start zk cluster -```bash -sbin/start_zks.sh -``` - -## Start HDFS cluster +## Start HDFS cluster (Optional) ```bash sbin/run_hdfs.sh ``` -## Start Kafka cluster +## Start GlusterFS cluster (Optional) ```bash -sbin/start_kafka.sh -``` - - -## Create kafka topic yao -```bash -# delete old -bin/kafka-topics.sh \ - --delete \ - --zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \ - --topic yao - -# create new -bin/kafka-topics.sh \ - --create \ - --zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \ - --replication-factor 3 \ - --partitions 1 \ - --topic yao +sbin/start_glusterfs.sh ```