From b363c63d44eef05a06f8f7ff7c471ac4392cc4e9 Mon Sep 17 00:00:00 2001
From: "Suren A. Chilingaryan" <csa@ipecompute2.ands.kit.edu>
Date: Wed, 11 Nov 2020 05:29:12 +0100
Subject: Support for ufo & jupyter notebooks

---
 build/buildah-astra.sh           | 29 +++------------------------
 build/buildah-base.sh            | 42 ++++++++++++++++++++++++++++++++++++++++
 build/buildah-ccpi.sh            | 15 ++------------
 build/buildah-remote.sh          | 20 +++++++++++++++++++
 build/buildah-tomo.sh            | 11 +++++++++++
 build/buildah-ufo.sh             | 11 +++++++++++
 build/buildah.sh                 |  3 +++
 build/extract.sh                 |  6 +++++-
 build/pre-setup/requirements.txt |  1 +
 build/setup/build/cmake.sh       |  2 +-
 build/setup/build/python.sh      |  2 +-
 build/setup/repos.sh             |  6 ++++++
 build/update.sh                  |  9 ++++++---
 run/config.sh                    |  8 ++++++++
 run/devel.sh                     |  8 ++++----
 run/run-devel.sh                 |  5 -----
 run/run-jupyter.sh               | 13 +++++++++++++
 run/run-ssh.sh                   | 10 +++++-----
 run/run.sh                       |  5 +++--
 run/update-from-devel.sh         |  8 +++++---
 20 files changed, 150 insertions(+), 64 deletions(-)
 create mode 100644 build/buildah-base.sh
 create mode 100644 build/buildah-remote.sh
 create mode 100644 build/buildah-tomo.sh
 create mode 100644 build/buildah-ufo.sh
 create mode 100644 run/config.sh
 delete mode 100644 run/run-devel.sh
 create mode 100644 run/run-jupyter.sh

diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh
index fa69db6..1282e28 100644
--- a/build/buildah-astra.sh
+++ b/build/buildah-astra.sh
@@ -2,32 +2,9 @@
 
 set -o errexit
 
-container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+container=$(buildah from pycuda:latest)
 
-buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
-
-buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
-buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
-buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
-buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
-
-buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
-buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
-buildah run $container sh -c 'apt-get install -y cmake autoconf automake libtool libboost-all-dev'
-
-buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
-buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
-
-buildah copy $container pre-setup /root/pre-setup
-buildah run $container pip3 install -r /root/pre-setup/requirements.txt
-
-buildah copy $container setup /root/setup
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
 buildah run $container bash /root/setup/repos.sh "astra"
+
 buildah commit --format docker $container astra:latest
diff --git a/build/buildah-base.sh b/build/buildah-base.sh
new file mode 100644
index 0000000..b656515
--- /dev/null
+++ b/build/buildah-base.sh
@@ -0,0 +1,42 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
+
+buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
+
+buildah run $container ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime
+buildah run $container sh -c 'echo "Europe/London" > /etc/timezone'
+
+buildah run $container sh -c 'echo "ssh:x:101:" >> /etc/group'
+buildah run $container sh -c 'echo "sshd:x:101:101:sshd:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "messagebus:x:102:" >> /etc/group'
+buildah run $container sh -c 'echo "messagebus:x:102:102:messagebus:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "rdma:x:103:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-journal:x:104:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-network:x:105:105:systemd-network:/dev/null:/sbin/nologin" >> /etc/passwd'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:" >> /etc/group'
+buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/dev/null:/sbin/nologin" >> /etc/passwd'
+
+buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
+buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
+buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev'
+buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev'
+buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy  python-gobject python-tifffile'
+buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo'
+buildah run $container sh -c 'apt-get install -y jupyter-notebook'
+
+buildah run $container sh -c 'update-alternatives --remove python /usr/bin/python2'
+buildah run $container sh -c 'update-alternatives --install /usr/bin/python python /usr/bin/python3 10'
+
+buildah run $container mkdir -p /etc/OpenCL/vendors
+buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd'
+
+buildah copy $container pre-setup /root/pre-setup
+buildah run $container pip3 install -r /root/pre-setup/requirements.txt
+
+buildah copy $container setup /root/setup
+buildah commit --format docker $container pycuda:latest
diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh
index 1bc6b10..3d006fc 100644
--- a/build/buildah-ccpi.sh
+++ b/build/buildah-ccpi.sh
@@ -4,18 +4,7 @@ set -o errexit
 
 container=$(buildah from astra:latest)
 
-buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data $container
-buildah config --port 2222 $container
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
+buildah run $container bash /root/setup/repos.sh "ccpi"
 
-buildah run $container bash /root/setup/repos.sh "all"
-
-buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
-#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
-
-buildah run $container sh -c 'service ssh start'
-buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
 buildah commit --format docker $container ccpi:latest
-
-echo $container
-
diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh
new file mode 100644
index 0000000..1016d19
--- /dev/null
+++ b/build/buildah-remote.sh
@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+container=$(buildah from ${image}:latest)
+buildah config --port 2222/tcp --port 8888/tcp $container
+
+buildah run $container sh -c "echo '$(id -ng):x:$(id -g):' >> /etc/group"
+#buildah run $container sh -c "echo '$(id -nu):x:$(id -u):$(id -g):$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+buildah run $container sh -c "echo '$(id -nu):x:0:0:$(id -nu):/ccpi/data:/bin/bash' >> /etc/passwd"
+
+buildah run $container sh -c 'service ssh start'
+buildah run $container sh -c "echo '$(id -nu):test' | chpasswd"
+
+buildah commit --format docker $container ${image}-remote:latest
+
+echo "$container build from ${image} and saved as ${image}-remote"
diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh
new file mode 100644
index 0000000..ed9287e
--- /dev/null
+++ b/build/buildah-tomo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from ccpi:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container tomo:latest
diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh
new file mode 100644
index 0000000..ee8e328
--- /dev/null
+++ b/build/buildah-ufo.sh
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+set -o errexit
+
+container=$(buildah from pycuda:latest)
+
+buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
+
+buildah run $container bash /root/setup/repos.sh "ufo"
+
+buildah commit --format docker $container ufo:latest
diff --git a/build/buildah.sh b/build/buildah.sh
index 2f4f283..8e84457 100644
--- a/build/buildah.sh
+++ b/build/buildah.sh
@@ -1,4 +1,7 @@
 set -o errexit
 
+bash buildah-base.sh
 bash buildah-astra.sh
 bash buildah-ccpi.sh
+bash buildah-tomo.sh
+bash buildah-remote.sh tomo
diff --git a/build/extract.sh b/build/extract.sh
index 5321db8..17c6e3f 100644
--- a/build/extract.sh
+++ b/build/extract.sh
@@ -2,7 +2,11 @@
 
 set -o errexit
 
-container=$(buildah from localhost/ccpi)
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+
+
+container=$(buildah from localhost/${image})
 path=$(buildah mount $container)
 mkdir -p ../repos/
 cp -ra "$path/ccpi/repos"/* ../repos/
diff --git a/build/pre-setup/requirements.txt b/build/pre-setup/requirements.txt
index 87e3dca..e7b8d3b 100644
--- a/build/pre-setup/requirements.txt
+++ b/build/pre-setup/requirements.txt
@@ -4,3 +4,4 @@ scipy
 matplotlib
 h5py
 pillow
+pycuda
diff --git a/build/setup/build/cmake.sh b/build/setup/build/cmake.sh
index 09beccf..1065907 100644
--- a/build/setup/build/cmake.sh
+++ b/build/setup/build/cmake.sh
@@ -4,6 +4,6 @@ set -o errexit
 
 pypath=$(python3 -c 'import site; print(site.getsitepackages()[0])')
 #rm CMakeCache.txt
-cmake -D "PYTHON_DEST_DIR:PATH=$pypath" .
+cmake -DCMAKE_INSTALL_PREFIX=/usr -D "PYTHON_DEST_DIR:PATH=$pypath" .
 make
 make install
diff --git a/build/setup/build/python.sh b/build/setup/build/python.sh
index de08a2c..3694a98 100644
--- a/build/setup/build/python.sh
+++ b/build/setup/build/python.sh
@@ -3,7 +3,7 @@
 set -o errexit
 
 (
-    cd Wrappers/Python
+    [ -d "Wrappers/Python" ] &&  cd "Wrappers/Python"
     python3 setup.py install
 )
 
diff --git a/build/setup/repos.sh b/build/setup/repos.sh
index 92c1a2f..0c7481a 100644
--- a/build/setup/repos.sh
+++ b/build/setup/repos.sh
@@ -20,6 +20,12 @@ repos_ccpi=( \
     "https://github.com/vais-ral/CCPi-Regularisation-Toolkit.git master ccpi.sh" \
 )
 
+repos_ufo=( \
+    "https://github.com/ufo-kit/ufo-core.git master cmake.sh" \
+    "https://github.com/ufo-kit/ufo-filters.git master cmake.sh" \
+    "https://github.com/ufo-kit/tofu master python.sh" \
+)
+
 repos_all=("${repos_astra[@]}" "${repos_ccpi[@]}")
 declare -n repos="repos_$repo_group"
 
diff --git a/build/update.sh b/build/update.sh
index 0201a4d..a3dc9ff 100644
--- a/build/update.sh
+++ b/build/update.sh
@@ -2,15 +2,18 @@
 
 set -o errexit
 
+image="$1"
+[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX>"; exit 1; }
+
 #container=cuda-working-container-15
-container=$(buildah from localhost/ccpi)
+container=$(buildah from localhost/${image})
 
 #buildah run $container bash
 #exit
 
-buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data  $container
+buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
 buildah copy $container setup /root/setup
 buildah run $container bash /root/setup/repos.sh "all" "/root/setup"
 
 #buildah commit --format docker $container ccpi:latest
-echo "buildah commit --format docker $container ccpi:latest"
+echo "buildah commit --format docker $container ${image}:latest"
diff --git a/run/config.sh b/run/config.sh
new file mode 100644
index 0000000..048ecfb
--- /dev/null
+++ b/run/config.sh
@@ -0,0 +1,8 @@
+port="$(($(id -u)%100 + 100))"
+ssh_port="${port}22"
+web_port="${port}80"
+
+image="$1"
+[ -n "$image" ] || image="tomo-remote"		# tomo-remote:latest tomo-remote:devel (also ccpi, ufo, tomo)
+
+echo "Image: $image, Ports: $ssh_port, $web_port"
diff --git a/run/devel.sh b/run/devel.sh
index ad8fa15..a15f203 100644
--- a/run/devel.sh
+++ b/run/devel.sh
@@ -2,10 +2,10 @@
 
 set -o errexit
 
+. config.sh
+
 if [ ! -d ../repos ]; then
-    buildah unshare bash ../build/extract.sh
+    buildah unshare bash ../build/extract.sh  ${image}
 fi
 
-port="$(($(id -u) - 900))22"
-
-podman run --name "ccpi-devel" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $port:2222/tcp -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash -c "bash /root/setup/provision.sh; bash"
+podman run --name "ccpi-devel" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $ssh_port:2222/tcp -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/${image} bash -c "bash /root/setup/provision.sh; bash"
diff --git a/run/run-devel.sh b/run/run-devel.sh
deleted file mode 100644
index 2adb95c..0000000
--- a/run/run-devel.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#! /bin/bash
-
-set -o errexit
-
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data localhost/ccpi:devel bash
diff --git a/run/run-jupyter.sh b/run/run-jupyter.sh
new file mode 100644
index 0000000..f596a2b
--- /dev/null
+++ b/run/run-jupyter.sh
@@ -0,0 +1,13 @@
+#! /bin/bash
+
+set -o errexit
+
+. config.sh
+
+echo "Running jupyter on the port $web_port"
+echo "Create ssh tunnel and use browser to connect http://localhost:$web_port"
+echo "    ssh -L $web_port:localhost:$web_port $(hostname)"
+echo "Generate password file '/ccpi/data/.jupyter/jupyter_notebook_config.json' with running in the container the following command"
+echo "    jupyter notebook password"
+
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${web_port}:8888/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} jupyter notebook --allow-root --no-browser --ip 0.0.0.0 --port=8888 --notebook-dir=/ccpi/data/ --debug
\ No newline at end of file
diff --git a/run/run-ssh.sh b/run/run-ssh.sh
index 950024a..c467334 100644
--- a/run/run-ssh.sh
+++ b/run/run-ssh.sh
@@ -2,12 +2,12 @@
 
 set -o errexit
 
-port="$(($(id -u) - 900))22"
+. config.sh
 
-echo "Running on the port $port"
+echo "Running ssh on the port $ssh_port"
 echo "Create ssh tunnel and connect:"
-echo "    ssh -L $port:localhost:$port $(hostname)"
+echo "    ssh -L $ssh_port:localhost:$ssh_port $(hostname)"
 echo " and connect locally "
-echo "    ssh -p $port localhost"
+echo "    ssh -p $ssh_port localhost"
 
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${port}:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/ccpi /usr/sbin/sshd -D -p 2222
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${ssh_port}:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} /usr/sbin/sshd -D -p 2222
diff --git a/run/run.sh b/run/run.sh
index 0010e1d..bdfade3 100644
--- a/run/run.sh
+++ b/run/run.sh
@@ -2,5 +2,6 @@
 
 set -o errexit
 
-port="$(($(id -u) - 900))22"
-podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p $port:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/ccpi bash
+. config.sh
+
+podman run --name "ccpi-run" -it --rm --hooks-dir /usr/share/containers/oci/hooks.d/ -p ${web_port}:8888/tcp -p $ssh_port:2222/tcp -v ~/ccpi/data:/ccpi/data localhost/${image} bash
diff --git a/run/update-from-devel.sh b/run/update-from-devel.sh
index 19aee6b..b2a8b25 100644
--- a/run/update-from-devel.sh
+++ b/run/update-from-devel.sh
@@ -2,11 +2,13 @@
 
 set -o errexit
 
+. config.sh
+
 if [ ! -d ../repos ]; then
-    buildah unshare bash ../build/extract.sh
+    buildah unshare bash ../build/extract.sh ${image}
 fi
 
-podman run --name "ccpi-devel" -it --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/ccpi bash /root/setup/provision.sh
+podman run --name "ccpi-devel" -it --hooks-dir /usr/share/containers/oci/hooks.d/ -v ~/ccpi/data:/ccpi/data -v ~/ccpi/repos:/ccpi/repos localhost/${image} bash /root/setup/provision.sh
 podman container cp ../repos/ ccpi-devel:/ccpi/
-podman container commit ccpi-devel ccpi:devel
+podman container commit ccpi-devel ${image}:devel
 podman rm ccpi-devel
-- 
cgit v1.2.1