From ffb131a0d358af4022f0dcbc6cf45f9bb54e3b94 Mon Sep 17 00:00:00 2001 From: Newnius Date: Mon, 4 May 2020 15:19:25 +0800 Subject: [PATCH] update --- tensorflow/1.14-gpu/Dockerfile | 4 +++- tensorflow/1.14-gpu/bootstrap.sh | 2 +- tensorflow/1.14-gpu/save.py | 20 ++++++++++++++++++++ tensorflow/2.1-gpu/Dockerfile | 2 ++ tensorflow/2.1-gpu/bootstrap.sh | 2 +- tensorflow/2.1-gpu/save.py | 20 ++++++++++++++++++++ 6 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 tensorflow/1.14-gpu/save.py create mode 100644 tensorflow/2.1-gpu/save.py diff --git a/tensorflow/1.14-gpu/Dockerfile b/tensorflow/1.14-gpu/Dockerfile index e486bf4..b9d8655 100644 --- a/tensorflow/1.14-gpu/Dockerfile +++ b/tensorflow/1.14-gpu/Dockerfile @@ -8,8 +8,10 @@ RUN apt update && \ apt install -y git && \ rm -rf /var/lib/apt/lists/* -RUN pip install requests psutil py-cpuinfo pandas +RUN pip install requests psutil py-cpuinfo pandas hdfs ADD bootstrap.sh /etc/bootstrap.sh +ADD save.py /etc/save.py + ENTRYPOINT ["/etc/bootstrap.sh"] diff --git a/tensorflow/1.14-gpu/bootstrap.sh b/tensorflow/1.14-gpu/bootstrap.sh index f4daf49..503e80b 100755 --- a/tensorflow/1.14-gpu/bootstrap.sh +++ b/tensorflow/1.14-gpu/bootstrap.sh @@ -18,4 +18,4 @@ fi eval $@ # persist output -sleep 15 +python /etc/save.py diff --git a/tensorflow/1.14-gpu/save.py b/tensorflow/1.14-gpu/save.py new file mode 100644 index 0000000..16ad890 --- /dev/null +++ b/tensorflow/1.14-gpu/save.py @@ -0,0 +1,20 @@ +from hdfs import * +import os +import time + + +if __name__ == '__main__': + os.environ["TZ"] = 'Asia/Shanghai' + if hasattr(time, 'tzset'): + time.tzset() + try: + hdfs_address = os.environ['hdfs_address'] + hdfs_dir = os.environ['hdfs_dir'] + output_dir = os.environ['output_dir'] + + client = Client(hdfs_address) + client.upload(hdfs_dir, output_dir) + + print('Save ' + output_dir + 'to' + hdfs_address + ' ' + hdfs_dir) + except Exception as e: + print('Unable to persist data to HDFS,', str(e)) diff --git a/tensorflow/2.1-gpu/Dockerfile b/tensorflow/2.1-gpu/Dockerfile index d6c003c..a617fd0 100644 --- a/tensorflow/2.1-gpu/Dockerfile +++ b/tensorflow/2.1-gpu/Dockerfile @@ -12,4 +12,6 @@ RUN pip3 install requests psutil py-cpuinfo pandas typing ADD bootstrap.sh /etc/bootstrap.sh +ADD save.py /etc/save.py + ENTRYPOINT ["/etc/bootstrap.sh"] diff --git a/tensorflow/2.1-gpu/bootstrap.sh b/tensorflow/2.1-gpu/bootstrap.sh index 23c98c3..b7b7a88 100755 --- a/tensorflow/2.1-gpu/bootstrap.sh +++ b/tensorflow/2.1-gpu/bootstrap.sh @@ -18,4 +18,4 @@ fi eval $@ # Persist output -sleep 15 +python /etc/save.py diff --git a/tensorflow/2.1-gpu/save.py b/tensorflow/2.1-gpu/save.py new file mode 100644 index 0000000..16ad890 --- /dev/null +++ b/tensorflow/2.1-gpu/save.py @@ -0,0 +1,20 @@ +from hdfs import * +import os +import time + + +if __name__ == '__main__': + os.environ["TZ"] = 'Asia/Shanghai' + if hasattr(time, 'tzset'): + time.tzset() + try: + hdfs_address = os.environ['hdfs_address'] + hdfs_dir = os.environ['hdfs_dir'] + output_dir = os.environ['output_dir'] + + client = Client(hdfs_address) + client.upload(hdfs_dir, output_dir) + + print('Save ' + output_dir + 'to' + hdfs_address + ' ' + hdfs_dir) + except Exception as e: + print('Unable to persist data to HDFS,', str(e))