update agent, add more metrics

Dockerfile

@@ -2,7 +2,7 @@ FROM quickdeploy/yao-python3
 
 MAINTAINER Newnius <newnius.cn@gmail.com>
 
-RUN pip3 install docker kafka
+RUN pip3 install docker kafka psutil
 
 ADD bootstrap.sh /etc/bootstrap.sh
 

README.md

@@ -1,31 +1,37 @@
 # YAO-agent
 
-
-```bash
-bin/kafka-topics.sh \
-	--describe \
-	--zookeeper zookeeper_node1:2181,zookeeper_node2:2181,zookeeper_node3:2181 \
-	--topic yao
-```
-
 ```bash
 bin/kafka-topics.sh \
 	--create \
-	--zookeeper zookeeper_node1:2181,zookeeper_node2:2181,zookeeper_node3:2181 \
+	--zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \
 	--replication-factor 3 \
 	--partitions 1 \
 	--topic yao
 ```
 
+```bash
+bin/kafka-topics.sh \
+	--describe \
+	--zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \
+	--topic yao
+```
+
 ```bash
 bin/kafka-console-consumer.sh \
-	--bootstrap-server kafka_node1:9091,kafka_node2:9092,kafka_node3:9093 \
+	--bootstrap-server kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \
 	--topic yao \
 	--from-beginning
 ```
 
 ```bash
 bin/kafka-console-producer.sh \
-	--broker-list kafka_node1:9091,kafka_node2:9092,kafka_node3:9093 \
+	--broker-list kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 \
+	--topic yao
+```
+
+```bash
+bin/kafka-topics.sh \
+	--delete \
+	--zookeeper zookeeper-node1:2181,zookeeper-node2:2181,zookeeper-node3:2181 \
 	--topic yao
 ```

bootstrap.sh

@@ -2,6 +2,9 @@
 
 # TODO: monitor the processes
 
+# run nvidia-smi in background to speed up the query and reduce CPU load (why?)
+nvidia-smi daemon
+
 python3 /root/monitor.py &
 
 python3 /root/executor.py &

executor.py

@@ -114,6 +114,26 @@ class MyHandler(BaseHTTPRequestHandler):
 			container.stop()
 			msg = {"code": 0}
 
+			self.send_response(200)
+			self.send_header('Content-type', 'application/json')
+			self.end_headers()
+			self.wfile.write(bytes(json.dumps(msg), "utf-8"))
+
+		elif self.path == "/remove":
+			form = cgi.FieldStorage(
+				fp=self.rfile,
+				headers=self.headers,
+				environ={
+					'REQUEST_METHOD': 'POST',
+					'CONTENT_TYPE': self.headers['Content-Type'],
+				})
+			container_id = form["id"].value
+
+			client = docker.from_env()
+			container = client.containers.get(container_id)
+			container.remove(force=True)
+			msg = {"code": 0}
+
 			self.send_response(200)
 			self.send_header('Content-type', 'application/json')
 			self.end_headers()

monitor.py

@@ -5,6 +5,9 @@ import json
 from xml.dom.minidom import parse
 import xml.dom.minidom
 from kafka import KafkaProducer
+import multiprocessing
+import psutil
+import math
 
 ClientID = os.getenv('ClientID', 1)
 ClientHost = os.getenv('ClientHost', "localhost")
@@ -71,7 +74,17 @@ def report_msg():
 
 		stats.append(stat)
 
-	post_fields = {'id': ClientID, 'host': ClientHost, 'status': stats}
+	mem = psutil.virtual_memory()
+
+	post_fields = {
+		'id': ClientID,
+		'host': ClientHost,
+		'status': stats,
+		'cpu_num': multiprocessing.cpu_count(),
+		'cpu_load': os.getloadavg()[0],
+		'mem_total': math.floor(mem.total / (1024. ** 3)),
+		'mem_available': math.floor(mem.available / (1024. ** 3))
+	}
 	data = json.dumps(post_fields)
 
 	producer = KafkaProducer(bootstrap_servers=KafkaBrokers)

test.py

@@ -36,7 +36,7 @@ def get_status(id):
 	container = client.containers.list(all=True, filters={'id': id})
 	status = {}
 	if len(container) > 0:
-		container= container[0]
+		container = container[0]
 		status['id'] = container.short_id
 		status['image'] = container.attrs['Config']['Image']
 		status['image_digest'] = container.attrs['Image']
@@ -47,6 +47,7 @@ def get_status(id):
 		if status['command'] is not None:
 			status['command'] = ' '.join(container.attrs['Config']['Cmd'])
 	print(status)
+	print(container.attrs)
 
 
 get_status('')