summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSuren A. Chilingaryan <csa@ipecompute4.ands.kit.edu>2022-09-08 02:16:52 +0200
committerSuren A. Chilingaryan <csa@ipecompute4.ands.kit.edu>2022-09-08 02:16:52 +0200
commitea2245b35cbfddb8c7473453b7d977b0a64f61b2 (patch)
treec97fb8c2e4c0f081eeb44cfb360f5d54c577a725
parenta3f77df3864464fa8cd6fdc613d2cf4bbae39124 (diff)
downloadccpi-ea2245b35cbfddb8c7473453b7d977b0a64f61b2.tar.gz
ccpi-ea2245b35cbfddb8c7473453b7d977b0a64f61b2.tar.bz2
ccpi-ea2245b35cbfddb8c7473453b7d977b0a64f61b2.tar.xz
ccpi-ea2245b35cbfddb8c7473453b7d977b0a64f61b2.zip
Update containers to ubuntu 22.04, gcc-12; include perf/gdb; add python libraries for opencv, machine learning, etc.
-rw-r--r--build/buildah-astra.sh7
-rw-r--r--build/buildah-base.sh26
-rw-r--r--build/buildah-ccpi.sh7
-rw-r--r--build/buildah-remote.sh11
-rw-r--r--build/buildah-tomo.sh7
-rw-r--r--build/buildah-ufo.sh7
-rw-r--r--build/buildah.sh12
-rw-r--r--build/update.sh12
-rw-r--r--patches/ccpi-readme.txt7
-rw-r--r--run/config.sh4
10 files changed, 74 insertions, 26 deletions
diff --git a/build/buildah-astra.sh b/build/buildah-astra.sh
index 1282e28..6f6e7b1 100644
--- a/build/buildah-astra.sh
+++ b/build/buildah-astra.sh
@@ -2,9 +2,12 @@
set -o errexit
-container=$(buildah from pycuda:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from pycuda:$tag)
buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
buildah run $container bash /root/setup/repos.sh "astra"
-buildah commit --format docker $container astra:latest
+buildah commit --format docker $container astra:$tag
diff --git a/build/buildah-base.sh b/build/buildah-base.sh
index 319ed6b..fcb3200 100644
--- a/build/buildah-base.sh
+++ b/build/buildah-base.sh
@@ -2,8 +2,13 @@
set -o errexit
+tag=latest
+[ -n "$1" ] && tag="$1"
+
#container=$(buildah from nvidia/cuda:10.1-devel-ubuntu18.04)
-container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04)
+[ $tag == "20.04" ] && container=$(buildah from nvidia/cuda:11.3.0-devel-ubuntu20.04)
+[ $tag == "22.04" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu22.04)
+[ -n "$container" ] && container=$(buildah from nvidia/cuda:11.7.1-devel-ubuntu$tag)
buildah config --label maintainer="Suren A. Chilingaryan <csa@suren.me>" $container
buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" $container
@@ -24,12 +29,23 @@ buildah run $container sh -c 'echo "systemd-resolve:x:106:106:systemd-resolve:/d
buildah run $container sh -c 'echo "systemd-timesync:x:107:" >> /etc/group'
buildah run $container sh -c 'echo "systemd-timesync:x:107:107:systemd-timesync:/dev/null:/sbin/nologin" >> /etc/passwd'
+# https://stackoverflow.com/questions/71941032/why-i-cannot-run-apt-update-inside-a-fresh-ubuntu22-04
+buildah run $container sh -c 'sed -i -e "s/^APT/# APT/" -e "s/^DPkg/# DPkg/" /etc/apt/apt.conf.d/docker-clean'
-buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc ca-certificates git openssh-server'
+buildah run $container sh -c 'apt-get update --fix-missing && apt-get install -y bash wget bzip2 mc rar ca-certificates git openssh-server'
buildah run $container sh -c 'apt-get install -y python3 python3-setuptools python3-pip cython3'
buildah run $container sh -c 'apt-get install -y cmake make autoconf automake libtool gcc g++ libboost-all-dev'
+
+[ $tag == "22.04" ] && buildah run $container sh -c 'apt-get install -y gcc-12 g++-12' # First gcc version supporting _Float16 data type
+[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y screen perf-tools-unstable gdb htop' # Debugging/Profilling tools
buildah run $container sh -c 'apt-get install -y ca-certificates pkg-config gobject-introspection liblapack-dev libjpeg-dev libtiff-dev libglib2.0-dev libjson-glib-dev libopenmpi-dev libhdf5-dev libclfft-dev libgsl-dev libgirepository1.0-dev fftw3-dev zlib1g-dev'
-buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy python-gobject tifffile'
+
+# Problems with python3-tifffile (this should be ran after installing pkgconfig/gobject stuff)
+buildah run $container sh -c 'sed -i -e "s/^post-invoke/# post-invoke/" /etc/dpkg/dpkg.cfg.d/pkg-config-hook-config'
+
+buildah run $container sh -c 'apt-get install -y python3-dev python3-numpy python3-gi python3-gi-cairo python3-tifffile'
+
+[ $tag != "20.04" ] && buildah run $container sh -c 'apt-get install -y python3-astropy python3-opencv python3-willow python3-pypillowfight python3-scipy python3-sklearn python3-skorch python3-tomopy' # Not needed for ccpi/ufo, but rather additional image-processing code for python
buildah run $container sh -c 'apt-get install -y ocl-icd-opencl-dev clinfo'
buildah run $container sh -c 'apt-get install -y jupyter-notebook'
@@ -39,8 +55,10 @@ buildah run $container sh -c 'update-alternatives --install /usr/bin/python pyth
buildah run $container mkdir -p /etc/OpenCL/vendors
buildah run $container sh -c 'echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd'
+[ $tag != "20.04" ] && buildah run $container sh -c 'rm /usr/bin/perf; ln -s /usr/lib/linux-tools/*/perf /usr/bin/perf'
+
buildah copy $container pre-setup /root/pre-setup
buildah run $container pip3 install -r /root/pre-setup/requirements.txt
buildah copy $container setup /root/setup
-buildah commit --format docker $container pycuda:latest
+buildah commit --format docker $container pycuda:$tag
diff --git a/build/buildah-ccpi.sh b/build/buildah-ccpi.sh
index 661e0ef..6912476 100644
--- a/build/buildah-ccpi.sh
+++ b/build/buildah-ccpi.sh
@@ -2,9 +2,12 @@
set -o errexit
-container=$(buildah from astra:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from astra:$tag)
buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env CIL_VERSION=21.0.0 --env HOME=/ccpi/data --env LD_LIBRARY_PATH="/ccpi/repos/CCPi-Regularisation-Toolkit/src/Core:/ccpi/repos/CCPi-Framework/src/Core" $container
buildah run $container bash /root/setup/repos.sh "ccpi"
-buildah commit --format docker $container ccpi:latest
+buildah commit --format docker $container ccpi:$tag
diff --git a/build/buildah-remote.sh b/build/buildah-remote.sh
index 5e9d038..7cb0eeb 100644
--- a/build/buildah-remote.sh
+++ b/build/buildah-remote.sh
@@ -3,9 +3,12 @@
set -o errexit
image="$1"
-[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX>"; exit 1; }
+[ -n "$image" ] || { echo "Usage: $0 <astra|ccpi|ufo|tomo|imageX> [tag]"; exit 1; }
-container=$(buildah from ${image}:latest)
+tag=latest
+[ -n "$2" ] && tag="$2"
+
+container=$(buildah from ${image}:${tag})
buildah config --port 2222/tcp --port 8888/tcp $container
buildah run $container sh -c 'apt-get install -y supervisor'
@@ -36,6 +39,6 @@ exitcodes=0
EOF
'
-buildah commit --format docker $container ${image}-remote:latest
+buildah commit --format docker $container ${image}-remote:${tag}
-echo "$container build from ${image} and saved as ${image}-remote"
+echo "$container build from ${image}:${tag} and saved as ${image}-remote:${tag}"
diff --git a/build/buildah-tomo.sh b/build/buildah-tomo.sh
index ed9287e..486f182 100644
--- a/build/buildah-tomo.sh
+++ b/build/buildah-tomo.sh
@@ -2,10 +2,13 @@
set -o errexit
-container=$(buildah from ccpi:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from ccpi:$tag)
buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
buildah run $container bash /root/setup/repos.sh "ufo"
-buildah commit --format docker $container tomo:latest
+buildah commit --format docker $container tomo:$tag
diff --git a/build/buildah-ufo.sh b/build/buildah-ufo.sh
index ee8e328..bdc6331 100644
--- a/build/buildah-ufo.sh
+++ b/build/buildah-ufo.sh
@@ -2,10 +2,13 @@
set -o errexit
-container=$(buildah from pycuda:latest)
+tag=latest
+[ -n "$1" ] && tag="$1"
+
+container=$(buildah from pycuda:$tag)
buildah config --env LANG="C.UTF-8" --env LC_ALL="C.UTF-8" --env TERM="xterm" --env HOME=/ccpi/data $container
buildah run $container bash /root/setup/repos.sh "ufo"
-buildah commit --format docker $container ufo:latest
+buildah commit --format docker $container ufo:$tag
diff --git a/build/buildah.sh b/build/buildah.sh
index 8e84457..2ba99e5 100644
--- a/build/buildah.sh
+++ b/build/buildah.sh
@@ -1,7 +1,9 @@
set -o errexit
-bash buildah-base.sh
-bash buildah-astra.sh
-bash buildah-ccpi.sh
-bash buildah-tomo.sh
-bash buildah-remote.sh tomo
+tag=22.04
+
+bash buildah-base.sh $tag
+bash buildah-astra.sh $tag
+bash buildah-ccpi.sh $tag
+bash buildah-tomo.sh $tag
+bash buildah-remote.sh tomo $tag
diff --git a/build/update.sh b/build/update.sh
index a3dc9ff..6a8ac7a 100644
--- a/build/update.sh
+++ b/build/update.sh
@@ -3,10 +3,14 @@
set -o errexit
image="$1"
-[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX>"; exit 1; }
+[ -n "$image" ] || { echo "Usage: $0 <astra|ufo|ccpi|ccpi-remote|tomo|tomo-remote|imageX> [tag]"; exit 1; }
+
+tag=latest
+[ -n "$2" ] && tag="$2"
+
#container=cuda-working-container-15
-container=$(buildah from localhost/${image})
+container=$(buildah from localhost/${image}:${tag})
#buildah run $container bash
#exit
@@ -15,5 +19,5 @@ buildah config --env CIL_VERSION=19.10 --env HOME=/ccpi/data --env LD_LIBRARY_PA
buildah copy $container setup /root/setup
buildah run $container bash /root/setup/repos.sh "all" "/root/setup"
-#buildah commit --format docker $container ccpi:latest
-echo "buildah commit --format docker $container ${image}:latest"
+#buildah commit --format docker $container ccpi:${tag}
+echo "buildah commit --format docker $container ${image}:${tag}"
diff --git a/patches/ccpi-readme.txt b/patches/ccpi-readme.txt
new file mode 100644
index 0000000..4399a22
--- /dev/null
+++ b/patches/ccpi-readme.txt
@@ -0,0 +1,7 @@
+ - There is also extra memory consumed in FISTA implementation, here:
+ /ccpi/repos/CCPi-Framework/Wrappers/Python/cil/optimisation/algorithms
+ We might reduce further 1-2 volumes if rewritten in C and temporarily variables
+ are properly eleminated. We further might introduce simplified memory manager
+ which will allow to give up further temporarily in FISTA and allocated them in
+ regularize and, then, vice-versa.
+
diff --git a/run/config.sh b/run/config.sh
index bc1d6c5..d2b01dd 100644
--- a/run/config.sh
+++ b/run/config.sh
@@ -6,7 +6,9 @@ web_port="${port}80"
image="$1"
-tag="latest"
+#tag="latest"
+tag="22.04"
+
[ -n "$image" ] || image="localhost/tomo-remote:$tag" # tomo-remote:latest tomo-remote:devel tomo-remote:evelina (also ccpi, ufo, tomo)
options="--cap-add SYS_ADMIN --cap-add=SYS_PTRACE --hooks-dir /usr/share/containers/oci/hooks.d"