add alluxio

alluxio/1.7/.dockerignore

@@ -0,0 +1 @@
+conf/.gitignore

alluxio/1.7/Dockerfile

@@ -0,0 +1,29 @@
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+FROM openjdk:8-jdk-alpine
+
+ARG ALLUXIO_TARBALL=http://downloads.alluxio.org/downloads/files/1.7.1/alluxio-1.7.1-bin.tar.gz
+
+RUN apk add --update bash && \
+    rm -rf /var/cache/apk/*
+
+ADD ${ALLUXIO_TARBALL} /opt/
+
+# if the tarball was remote, it needs to be untarred
+RUN cd /opt && \
+    (if ls | grep -q ".tar.gz"; then tar -xzf *.tar.gz && rm *.tar.gz; fi) && \
+    mv alluxio-* alluxio
+
+COPY conf /opt/alluxio/conf/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

alluxio/1.7/Dockerfile.fuse

@@ -0,0 +1,39 @@
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+FROM ubuntu:16.04
+
+ARG ALLUXIO_TARBALL=http://downloads.alluxio.org/downloads/files/1.7.0/alluxio-1.7.0-bin.tar.gz
+ENV ENABLE_FUSE true
+
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common && \
+    add-apt-repository -y ppa:openjdk-r/ppa && \
+	apt-get update && \
+	apt-get install -y --no-install-recommends \
+	openjdk-8-jdk openjdk-8-jre-headless libfuse-dev && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+
+RUN mkdir -p /alluxio-fuse
+
+ADD ${ALLUXIO_TARBALL} /opt/
+
+# if the tarball was remote, it needs to be untarred
+RUN cd /opt && \
+    (if ls | grep -q ".tar.gz"; then tar -xzf *.tar.gz && rm *.tar.gz; fi) && \
+    mv alluxio-* alluxio
+
+COPY conf /opt/alluxio/conf/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

alluxio/1.7/README.md

@@ -0,0 +1,71 @@
+## Building docker image
+To build the Alluxio Docker image from the default remote url, run
+
+```bash
+docker build -t alluxio .
+```
+
+To build with a local Alluxio tarball, specify the `ALLUXIO_TARBALL` build argument
+
+```bash
+docker build -t alluxio --build-arg ALLUXIO_TARBALL=alluxio-${version}.tar.gz .
+```
+
+## Running docker image
+The generated image expects to be run with single argument of "master", "worker", or "proxy".
+To set an Alluxio configuration property, convert it to an environment variable by uppercasing
+and replacing periods with underscores. For example, `alluxio.master.hostname` converts to
+`ALLUXIO_MASTER_HOSTNAME`. You can then set the environment variable on the image with
+`-e PROPERTY=value`. Alluxio configuration values will be copied to `conf/alluxio-site.properties`
+when the image starts.
+
+```bash
+docker run -e ALLUXIO_MASTER_HOSTNAME=ec2-203-0-113-25.compute-1.amazonaws.com alluxio [master|worker|proxy]
+```
+
+Additional configuration files can be included when building the image by adding them to the
+`integration/docker/conf/` directory. All contents of this directory will be
+copied to `/opt/alluxio/conf`.
+
+
+## Building docker image with FUSE support
+To build a docker image with 
+[FUSE](https://www.alluxio.org/docs/master/en/Mounting-Alluxio-FS-with-FUSE.html) support, run
+
+```bash
+docker build -f Dockerfile.fuse -t alluxio-fuse .
+```
+
+You can add the same arguments supported by the non-FUSE docker file.
+
+
+## Running docker image with FUSE support
+There are a couple extra arguments required to run the docker image with FUSE support, for example:
+
+```bash
+docker run -e ALLUXIO_MASTER_HOSTNAME=alluxio-master --cap-add SYS_ADMIN --device /dev/fuse  alluxio-fuse [master|worker|proxy]
+```
+
+Note: running FUSE in docker requires adding 
+[SYS_ADMIN capability](http://man7.org/linux/man-pages/man7/capabilities.7.html) to the container. 
+This removes isolation of the container and should be used with caution.
+
+## Extending docker image with applications
+You can easily extend the docker image to include applications to run on top of Alluxio. 
+In order for the application to access data from Alluxio storage mounted with FUSE, it must run
+in the same container as Alluxio. For example, to run TensorFlow with Alluxio inside a docker 
+container, just edit `Dockerfile.fuse` and replace 
+
+```bash
+FROM ubuntu:16.04
+```
+
+with
+
+```bash
+FROM tensorflow/tensorflow:1.3.0
+```
+
+You can then build the image with the same command for building image with FUSE support and run it.
+There is a pre-built docker image with TensorFlow at 
+https://hub.docker.com/r/alluxio/alluxio-tensorflow/ 

alluxio/1.7/bin/alluxio-master.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+set -e
+
+SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd)
+
+USAGE="Usage: alluxio-master
+
+alluxio-master launches the Alluxio master in the foreground and logs to both
+the console and default log file. To configure the master, add configuration
+properties in alluxio-site.properties or environment variables in
+alluxio-env.sh."
+
+if [[ "$#" -gt "0" ]]; then
+  echo "${USAGE}"
+  exit 0
+fi
+
+# Log to both the console and the master logs file
+ALLUXIO_MASTER_LOGGER="Console,MASTER_LOGGER"
+
+. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh
+
+${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_MASTER_JAVA_OPTS} alluxio.master.AlluxioMaster

alluxio/1.7/bin/alluxio-proxy.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+set -e
+
+SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd)
+
+USAGE="Usage: alluxio-proxy
+
+alluxio-proxy launches the Alluxio proxy in the foreground and logs to both the
+console and default log file. The proxy will provide a REST interface for
+interacting with the Alluxio filesystem. To configure the proxy, add
+configuration properties in alluxio-site.properties or environment variables in
+alluxio-env.sh."
+
+if [[ "$#" -gt "0" ]]; then
+  echo "${USAGE}"
+  exit 0
+fi
+
+# Log to both the console and the proxy logs file
+ALLUXIO_PROXY_LOGGER="Console,PROXY_LOGGER"
+
+. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh
+
+${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_PROXY_JAVA_OPTS} alluxio.proxy.AlluxioProxy

alluxio/1.7/bin/alluxio-worker.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+set -e
+
+SCRIPT_DIR=$(cd "$( dirname "$0" )"; pwd)
+
+USAGE="Usage: alluxio-worker
+
+alluxio-worker launches the Alluxio worker in the foreground and logs to both
+the console and default log file. To configure the worker, add configuration
+properties in alluxio-site.properties or environment variables in
+alluxio-env.sh."
+
+if [[ "$#" -gt "0" ]]; then
+  echo "${USAGE}"
+  exit 0
+fi
+
+# Log to both the console and the worker logs file
+ALLUXIO_WORKER_LOGGER="Console,WORKER_LOGGER"
+
+. ${SCRIPT_DIR}/../../../libexec/alluxio-config.sh
+
+${JAVA} -cp ${ALLUXIO_SERVER_CLASSPATH} ${ALLUXIO_WORKER_JAVA_OPTS} alluxio.worker.AlluxioWorker

alluxio/1.7/conf/alluxio-env.sh.template

@@ -0,0 +1 @@
+../../../conf/alluxio-env.sh.template

alluxio/1.7/conf/alluxio-site.properties.template

@@ -0,0 +1 @@
+../../../conf/alluxio-site.properties.template

alluxio/1.7/entrypoint.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+#
+# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+# (the "License"). You may not use this work except in compliance with the License, which is
+# available at www.apache.org/licenses/LICENSE-2.0
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied, as more fully set forth in the License.
+#
+# See the NOTICE file distributed with this work for information regarding copyright ownership.
+#
+
+set -e
+
+NO_FORMAT='--no-format'
+
+function printUsage {
+  echo "Usage: COMMAND [COMMAND_OPTIONS]"
+  echo
+  echo "COMMAND is one of:"
+  echo -e " master [--no-format]    \t Start Alluxio master. If --no-format is specified, do not format"
+  echo -e " worker [--no-format]    \t Start Alluxio worker. If --no-format is specified, do not format"
+  echo -e " proxy                   \t Start Alluxio proxy"
+}
+
+if [[ $# -lt 1 ]]; then
+  printUsage
+  exit 1
+fi
+
+service=$1
+options=$2
+
+# Only set ALLUXIO_RAM_FOLDER if tiered storage isn't explicitly configured
+if [[ -z "${ALLUXIO_WORKER_TIEREDSTORE_LEVEL0_DIRS_PATH}" ]]; then
+  # Docker will set this tmpfs up by default. Its size is configurable through the
+  # --shm-size argument to docker run
+  export ALLUXIO_RAM_FOLDER=${ALLUXIO_RAM_FOLDER:-/dev/shm}
+fi
+
+home=/opt/alluxio
+cd ${home}
+
+# List of environment variables which go in alluxio-env.sh instead of
+# alluxio-site.properties
+alluxio_env_vars=(
+  ALLUXIO_CLASSPATH
+  ALLUXIO_HOSTNAME
+  ALLUXIO_JARS
+  ALLUXIO_JAVA_OPTS
+  ALLUXIO_MASTER_JAVA_OPTS
+  ALLUXIO_PROXY_JAVA_OPTS
+  ALLUXIO_RAM_FOLDER
+  ALLUXIO_USER_JAVA_OPTS
+  ALLUXIO_WORKER_JAVA_OPTS
+)
+
+for keyvaluepair in $(env); do
+  # split around the "="
+  key=$(echo ${keyvaluepair} | cut -d= -f1)
+  value=$(echo ${keyvaluepair} | cut -d= -f2-)
+  if [[ "${alluxio_env_vars[*]}" =~ "${key}" ]]; then
+    echo "export ${key}=${value}" >> conf/alluxio-env.sh
+  else
+    # check if property name is valid
+    if confkey=$(bin/alluxio runClass alluxio.cli.GetConfKey ${key} 2> /dev/null); then
+      echo "${confkey}=${value}" >> conf/alluxio-site.properties
+    fi
+  fi
+done
+
+if [ "$ENABLE_FUSE" = true ]; then
+  integration/fuse/bin/alluxio-fuse mount /alluxio-fuse /
+fi
+
+case ${service,,} in
+  master)
+    if [[ -n ${options} && ${options} != ${NO_FORMAT} ]]; then
+      printUsage
+      exit 1
+    fi
+    if [[ ${options} != ${NO_FORMAT} ]]; then
+      bin/alluxio formatMaster
+    fi
+    integration/docker/bin/alluxio-master.sh
+    ;;
+  worker)
+    if [[ -n ${options} && ${options} != ${NO_FORMAT} ]]; then
+      printUsage
+      exit 1
+    fi
+    if [[ ${options} != ${NO_FORMAT} ]]; then
+      bin/alluxio formatWorker
+    fi
+    integration/docker/bin/alluxio-worker.sh
+    ;;
+  proxy)
+    integration/docker/bin/alluxio-proxy.sh
+    ;;
+  *)
+    printUsage
+    exit 1
+    ;;
+esac