From 8a8e018d9dad08efebe81a7c9082de84d14fb2a2 Mon Sep 17 00:00:00 2001 From: Newnius Date: Tue, 19 Jun 2018 16:01:00 +0800 Subject: [PATCH] add alluxio --- alluxio/1.7/.dockerignore | 1 + alluxio/1.7/Dockerfile | 29 +++++ alluxio/1.7/Dockerfile.fuse | 39 +++++++ alluxio/1.7/README.md | 71 ++++++++++++ alluxio/1.7/bin/alluxio-master.sh | 34 ++++++ alluxio/1.7/bin/alluxio-proxy.sh | 35 ++++++ alluxio/1.7/bin/alluxio-worker.sh | 34 ++++++ alluxio/1.7/conf/alluxio-env.sh.template | 1 + .../1.7/conf/alluxio-site.properties.template | 1 + alluxio/1.7/entrypoint.sh | 104 ++++++++++++++++++ 10 files changed, 349 insertions(+) create mode 100644 alluxio/1.7/.dockerignore create mode 100644 alluxio/1.7/Dockerfile create mode 100644 alluxio/1.7/Dockerfile.fuse create mode 100644 alluxio/1.7/README.md create mode 100755 alluxio/1.7/bin/alluxio-master.sh create mode 100755 alluxio/1.7/bin/alluxio-proxy.sh create mode 100755 alluxio/1.7/bin/alluxio-worker.sh create mode 120000 alluxio/1.7/conf/alluxio-env.sh.template create mode 120000 alluxio/1.7/conf/alluxio-site.properties.template create mode 100755 alluxio/1.7/entrypoint.sh diff --git a/alluxio/1.7/.dockerignore b/alluxio/1.7/.dockerignore new file mode 100644 index 0000000..71c1e16 --- /dev/null +++ b/alluxio/1.7/.dockerignore @@ -0,0 +1 @@ +conf/.gitignore diff --git a/alluxio/1.7/Dockerfile b/alluxio/1.7/Dockerfile new file mode 100644 index 0000000..dd286b5 --- /dev/null +++ b/alluxio/1.7/Dockerfile @@ -0,0 +1,29 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +FROM openjdk:8-jdk-alpine + +ARG ALLUXIO_TARBALL=http://downloads.alluxio.org/downloads/files/1.7.1/alluxio-1.7.1-bin.tar.gz + +RUN apk add --update bash && \ + rm -rf /var/cache/apk/* + +ADD ${ALLUXIO_TARBALL} /opt/ + +# if the tarball was remote, it needs to be untarred +RUN cd /opt && \ + (if ls | grep -q ".tar.gz"; then tar -xzf *.tar.gz && rm *.tar.gz; fi) && \ + mv alluxio-* alluxio + +COPY conf /opt/alluxio/conf/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/alluxio/1.7/Dockerfile.fuse b/alluxio/1.7/Dockerfile.fuse new file mode 100644 index 0000000..12ce0ad --- /dev/null +++ b/alluxio/1.7/Dockerfile.fuse @@ -0,0 +1,39 @@ +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +FROM ubuntu:16.04 + +ARG ALLUXIO_TARBALL=http://downloads.alluxio.org/downloads/files/1.7.0/alluxio-1.7.0-bin.tar.gz +ENV ENABLE_FUSE true + +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \ + add-apt-repository -y ppa:openjdk-r/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + openjdk-8-jdk openjdk-8-jre-headless libfuse-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + +RUN mkdir -p /alluxio-fuse + +ADD ${ALLUXIO_TARBALL} /opt/ + +# if the tarball was remote, it needs to be untarred +RUN cd /opt && \ + (if ls | grep -q ".tar.gz"; then tar -xzf *.tar.gz && rm *.tar.gz; fi) && \ + mv alluxio-* alluxio + +COPY conf /opt/alluxio/conf/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/alluxio/1.7/README.md b/alluxio/1.7/README.md new file mode 100644 index 0000000..e5c0da1 --- /dev/null +++ b/alluxio/1.7/README.md @@ -0,0 +1,71 @@ +## Building docker image +To build the Alluxio Docker image from the default remote url, run + +```bash +docker build -t alluxio . +``` + +To build with a local Alluxio tarball, specify the `ALLUXIO_TARBALL` build argument + +```bash +docker build -t alluxio --build-arg ALLUXIO_TARBALL=alluxio-${version}.tar.gz . +``` + +## Running docker image +The generated image expects to be run with single argument of "master", "worker", or "proxy". +To set an Alluxio configuration property, convert it to an environment variable by uppercasing +and replacing periods with underscores. For example, `alluxio.master.hostname` converts to +`ALLUXIO_MASTER_HOSTNAME`. You can then set the environment variable on the image with +`-e PROPERTY=value`. Alluxio configuration values will be copied to `conf/alluxio-site.properties` +when the image starts. + +```bash +docker run -e ALLUXIO_MASTER_HOSTNAME=ec2-203-0-113-25.compute-1.amazonaws.com alluxio [master|worker|proxy] +``` + +Additional configuration files can be included when building the image by adding them to the +`integration/docker/conf/` directory. All contents of this directory will be +copied to `/opt/alluxio/conf`. + + +## Building docker image with FUSE support +To build a docker image with +[FUSE](https://www.alluxio.org/docs/master/en/Mounting-Alluxio-FS-with-FUSE.html) support, run + +```bash +docker build -f Dockerfile.fuse -t alluxio-fuse . +``` + +You can add the same arguments supported by the non-FUSE docker file. + + +## Running docker image with FUSE support +There are a couple extra arguments required to run the docker image with FUSE support, for example: + +```bash +docker run -e ALLUXIO_MASTER_HOSTNAME=alluxio-master --cap-add SYS_ADMIN --device /dev/fuse alluxio-fuse [master|worker|proxy] +``` + +Note: running FUSE in docker requires adding +[SYS_ADMIN capability](http://man7.org/linux/man-pages/man7/capabilities.7.html) to the container. +This removes isolation of the container and should be used with caution. + +## Extending docker image with applications +You can easily extend the docker image to include applications to run on top of Alluxio. +In order for the application to access data from Alluxio storage mounted with FUSE, it must run +in the same container as Alluxio. For example, to run TensorFlow with Alluxio inside a docker +container, just edit `Dockerfile.fuse` and replace + +```bash +FROM ubuntu:16.04 +``` + +with + +```bash +FROM tensorflow/tensorflow:1.3.0 +``` + +You can then build the image with the same command for building image with FUSE support and run it. +There is a pre-built docker image with TensorFlow at +https://hub.docker.com/r/alluxio/alluxio-tensorflow/ diff --git a/alluxio/1.7/bin/alluxio-master.sh b/alluxio/1.7/bin/alluxio-master.sh new file mode 100755 index 0000000..fdedd5d --- /dev/null +++ b/alluxio/1.7/bin/alluxio-master.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -e + +SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd) + +USAGE="Usage: alluxio-master + +alluxio-master launches the Alluxio master in the foreground and logs to both +the console and default log file. To configure the master, add configuration +properties in alluxio-site.properties or environment variables in +alluxio-env.sh." + +if [[ "$#" -gt "0" ]]; then + echo "${USAGE}" + exit 0 +fi + +# Log to both the console and the master logs file +ALLUXIO_MASTER_LOGGER="Console,MASTER_LOGGER" + +. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh + +${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.AlluxioMaster diff --git a/alluxio/1.7/bin/alluxio-proxy.sh b/alluxio/1.7/bin/alluxio-proxy.sh new file mode 100755 index 0000000..f70a8c4 --- /dev/null +++ b/alluxio/1.7/bin/alluxio-proxy.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -e + +SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd) + +USAGE="Usage: alluxio-proxy + +alluxio-proxy launches the Alluxio proxy in the foreground and logs to both the +console and default log file. The proxy will provide a REST interface for +interacting with the Alluxio filesystem. To configure the proxy, add +configuration properties in alluxio-site.properties or environment variables in +alluxio-env.sh." + +if [[ "$#" -gt "0" ]]; then + echo "${USAGE}" + exit 0 +fi + +# Log to both the console and the proxy logs file +ALLUXIO_PROXY_LOGGER="Console,PROXY_LOGGER" + +. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh + +${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_PROXY_JAVA_OPTS} alluxio.proxy.AlluxioProxy diff --git a/alluxio/1.7/bin/alluxio-worker.sh b/alluxio/1.7/bin/alluxio-worker.sh new file mode 100755 index 0000000..6cdc07a --- /dev/null +++ b/alluxio/1.7/bin/alluxio-worker.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -e + +SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd) + +USAGE="Usage: alluxio-worker + +alluxio-worker launches the Alluxio worker in the foreground and logs to both +the console and default log file. To configure the worker, add configuration +properties in alluxio-site.properties or environment variables in +alluxio-env.sh." + +if [[ "$#" -gt "0" ]]; then + echo "${USAGE}" + exit 0 +fi + +# Log to both the console and the worker logs file +ALLUXIO_WORKER_LOGGER="Console,WORKER_LOGGER" + +. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh + +${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_WORKER_JAVA_OPTS} alluxio.worker.AlluxioWorker diff --git a/alluxio/1.7/conf/alluxio-env.sh.template b/alluxio/1.7/conf/alluxio-env.sh.template new file mode 120000 index 0000000..3e08257 --- /dev/null +++ b/alluxio/1.7/conf/alluxio-env.sh.template @@ -0,0 +1 @@ +../../../conf/alluxio-env.sh.template \ No newline at end of file diff --git a/alluxio/1.7/conf/alluxio-site.properties.template b/alluxio/1.7/conf/alluxio-site.properties.template new file mode 120000 index 0000000..aa0e449 --- /dev/null +++ b/alluxio/1.7/conf/alluxio-site.properties.template @@ -0,0 +1 @@ +../../../conf/alluxio-site.properties.template \ No newline at end of file diff --git a/alluxio/1.7/entrypoint.sh b/alluxio/1.7/entrypoint.sh new file mode 100755 index 0000000..e2bbd22 --- /dev/null +++ b/alluxio/1.7/entrypoint.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -e + +NO_FORMAT='--no-format' + +function printUsage { + echo "Usage: COMMAND [COMMAND_OPTIONS]" + echo + echo "COMMAND is one of:" + echo -e " master [--no-format] \t Start Alluxio master. If --no-format is specified, do not format" + echo -e " worker [--no-format] \t Start Alluxio worker. If --no-format is specified, do not format" + echo -e " proxy \t Start Alluxio proxy" +} + +if [[ $# -lt 1 ]]; then + printUsage + exit 1 +fi + +service=$1 +options=$2 + +# Only set ALLUXIO_RAM_FOLDER if tiered storage isn't explicitly configured +if [[ -z "${ALLUXIO_WORKER_TIEREDSTORE_LEVEL0_DIRS_PATH}" ]]; then + # Docker will set this tmpfs up by default. Its size is configurable through the + # --shm-size argument to docker run + export ALLUXIO_RAM_FOLDER=${ALLUXIO_RAM_FOLDER:-/dev/shm} +fi + +home=/opt/alluxio +cd ${home} + +# List of environment variables which go in alluxio-env.sh instead of +# alluxio-site.properties +alluxio_env_vars=( + ALLUXIO_CLASSPATH + ALLUXIO_HOSTNAME + ALLUXIO_JARS + ALLUXIO_JAVA_OPTS + ALLUXIO_MASTER_JAVA_OPTS + ALLUXIO_PROXY_JAVA_OPTS + ALLUXIO_RAM_FOLDER + ALLUXIO_USER_JAVA_OPTS + ALLUXIO_WORKER_JAVA_OPTS +) + +for keyvaluepair in $(env); do + # split around the "=" + key=$(echo ${keyvaluepair} | cut -d= -f1) + value=$(echo ${keyvaluepair} | cut -d= -f2-) + if [[ "${alluxio_env_vars[*]}" =~ "${key}" ]]; then + echo "export ${key}=${value}" >> conf/alluxio-env.sh + else + # check if property name is valid + if confkey=$(bin/alluxio runClass alluxio.cli.GetConfKey ${key} 2> /dev/null); then + echo "${confkey}=${value}" >> conf/alluxio-site.properties + fi + fi +done + +if [ "$ENABLE_FUSE" = true ]; then + integration/fuse/bin/alluxio-fuse mount /alluxio-fuse / +fi + +case ${service,,} in + master) + if [[ -n ${options} && ${options} != ${NO_FORMAT} ]]; then + printUsage + exit 1 + fi + if [[ ${options} != ${NO_FORMAT} ]]; then + bin/alluxio formatMaster + fi + integration/docker/bin/alluxio-master.sh + ;; + worker) + if [[ -n ${options} && ${options} != ${NO_FORMAT} ]]; then + printUsage + exit 1 + fi + if [[ ${options} != ${NO_FORMAT} ]]; then + bin/alluxio formatWorker + fi + integration/docker/bin/alluxio-worker.sh + ;; + proxy) + integration/docker/bin/alluxio-proxy.sh + ;; + *) + printUsage + exit 1 + ;; +esac