From 81001835ad8f83b3426daf67332ad7530efac37c Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <oscar.dominguez.ortiz@intel.com>
Date: Jun 21 2022 05:41:25 +0000
Subject: [PATCH 1/2] Add Containerfiles & script for periodc builds & rpmdb test


---

diff --git a/Containerfile.c8s b/Containerfile.c8s
new file mode 100644
index 0000000..71a1634
--- /dev/null
+++ b/Containerfile.c8s
@@ -0,0 +1,60 @@
+FROM quay.io/centoshyperscale/centos:stream8
+
+# Based on https://developers.redhat.com/blog/2019/08/14/best-practices-for-running-buildah-in-a-container
+# https://github.com/containers/buildah/blob/main/docs/tutorials/05-openshift-rootless-build.md
+# https://github.com/containers/buildah/blob/main/contrib/buildahimage/stable/Containerfile
+
+ENV _BUILDAH_STARTED_IN_USERNS=""
+ENV BUILDAH_ISOLATION=chroot
+ENV STORAGE_DRIVER=vfs
+ENV VERSION=8
+# Date for next build
+ENV DATE="08:00 next Fri"
+
+# Add user for rootless build
+RUN adduser build
+
+RUN dnf -y update
+RUN dnf -y install vi buildah podman
+RUN dnf -y clean all && \
+    rm -rf /var/cache /var/log/dnf*
+
+RUN chmod u-s /usr/bin/newuidmap && \
+    chmod u-s /usr/bin/newgidmap
+
+# Define uid/gid ranges for our user
+# https://github.com/containers/buildah/issues/3053
+RUN touch /etc/subgid /etc/subuid && \
+    chmod g=u /etc/subgid /etc/subuid /etc/passwd && \
+    echo build:10000:65536 > /etc/subuid && \
+    echo build:10000:65536 > /etc/subgid
+
+RUN (echo '[engine]';echo 'cgroup_manager = "cgroupfs"') > /etc/containers/containers.conf
+RUN chmod 644 /etc/containers/containers.conf
+
+RUN echo "export BUILDAH_ISOLATION=chroot" >> /home/build/.bashrc
+RUN mkdir -p /home/build/.local/share/containers
+
+# Adjust storage.conf to enable VFS since fuse dosen't work on Openshift
+RUN mkdir -p /home/build/.config/containers && \
+    (echo '[storage]';echo 'driver = "vfs"') > /home/build/.config/containers/storage.conf
+
+RUN mkdir -p /var/lib/shared/vfs-images \
+             /var/lib/shared/vfs-layers && \
+    touch /var/lib/shared/vfs-images/images.lock && \
+    touch /var/lib/shared/vfs-layers/layers.lock
+
+COPY make-hyperscale-container.sh /home/build
+COPY runner.sh /home/build
+RUN chmod a+x /home/build/make-hyperscale-container.sh
+RUN chmod a+x /home/build/runner.sh
+
+RUN chown build:build -R /home/build
+
+VOLUME /var/lib/containers
+VOLUME /home/build/.local/share/containers
+
+USER build
+WORKDIR /home/build
+
+ENTRYPOINT ["./runner.sh"]
diff --git a/Containerfile.c9s b/Containerfile.c9s
new file mode 100644
index 0000000..ae5d5a2
--- /dev/null
+++ b/Containerfile.c9s
@@ -0,0 +1,60 @@
+FROM quay.io/centoshyperscale/centos:stream9
+
+# Based on https://developers.redhat.com/blog/2019/08/14/best-practices-for-running-buildah-in-a-container
+# https://github.com/containers/buildah/blob/main/docs/tutorials/05-openshift-rootless-build.md
+# https://github.com/containers/buildah/blob/main/contrib/buildahimage/stable/Containerfile
+
+ENV _BUILDAH_STARTED_IN_USERNS=""
+ENV BUILDAH_ISOLATION=chroot
+ENV STORAGE_DRIVER=vfs
+ENV VERSION=9
+# Date for next build
+ENV DATE="08:00 next Fri"
+
+# Add user for rootless build
+RUN adduser build
+
+RUN dnf -y update
+RUN dnf -y install vi buildah
+RUN dnf -y clean all && \
+    rm -rf /var/cache /var/log/dnf*
+
+RUN chmod u-s /usr/bin/newuidmap && \
+    chmod u-s /usr/bin/newgidmap
+
+# Define uid/gid ranges for our user
+# https://github.com/containers/buildah/issues/3053
+RUN touch /etc/subgid /etc/subuid && \
+    chmod g=u /etc/subgid /etc/subuid /etc/passwd && \
+    echo build:10000:65536 > /etc/subuid && \
+    echo build:10000:65536 > /etc/subgid
+
+RUN (echo '[engine]';echo 'cgroup_manager = "cgroupfs"') > /etc/containers/containers.conf
+RUN chmod 644 /etc/containers/containers.conf
+
+RUN echo "export BUILDAH_ISOLATION=chroot" >> /home/build/.bashrc
+RUN mkdir -p /home/build/.local/share/containers
+
+# Adjust storage.conf to enable VFS since fuse dosen't work on Openshift
+RUN mkdir -p /home/build/.config/containers && \
+    (echo '[storage]';echo 'driver = "vfs"') > /home/build/.config/containers/storage.conf
+
+RUN mkdir -p /var/lib/shared/vfs-images \
+             /var/lib/shared/vfs-layers && \
+    touch /var/lib/shared/vfs-images/images.lock && \
+    touch /var/lib/shared/vfs-layers/layers.lock
+
+COPY make-hyperscale-container.sh /home/build
+COPY runner.sh /home/build
+RUN chmod a+x /home/build/make-hyperscale-container.sh
+RUN chmod a+x /home/build/runner.sh
+
+RUN chown build:build -R /home/build
+
+VOLUME /var/lib/containers
+VOLUME /home/build/.local/share/containers
+
+USER build
+WORKDIR /home/build
+
+ENTRYPOINT ["./runner.sh"]
diff --git a/runner.sh b/runner.sh
new file mode 100644
index 0000000..fb5e2bb
--- /dev/null
+++ b/runner.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+export TZ='America/Los_Angeles'
+
+# Hyperscale Centos Stream version, default 8
+VERSION="${VERSION:-8}"
+
+# Date for next build
+DATE="${DATE:-'8:00 next Fri'}"
+
+# Push to quay.io
+Push () {
+  buildah login -u $USERNAME -p $PASSWORD quay.io
+  buildah tag centos-stream-hyperscale-${VERSION} quay.io/centoshyperscale/centos:stream${VERSION}
+  buildah push quay.io/centoshyperscale/centos:stream${VERSION}
+  buildah logout quay.io
+}
+
+# Always run once immediately for easy testing.
+./make-hyperscale-container.sh $VERSION
+Push
+
+while true; do
+  TIME=$(($(date -d "$DATE" +%s) - $(date +%s)))
+  echo "Next build at $(date -d "$DATE")"
+  sleep "$TIME"
+  echo "Starting at $(date)"
+  ./make-hyperscale-container.sh $VERSION
+  Push
+  echo "Done at $(date)"
+  echo
+done
diff --git a/test_rpmdb.sh b/test_rpmdb.sh
new file mode 100644
index 0000000..4814116
--- /dev/null
+++ b/test_rpmdb.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -eu
+
+# rpmdb files for each rpmdb backend
+declare -A RPMDB_FILES
+RPMDB_FILES[ndb]="Packages.db"
+RPMDB_FILES[sqlite]="rpmdb.sqlite"
+RPMDB_FILES[bdb]="Packages"
+
+RPMDB_PATH=$(rpm -E "%{_dbpath}")
+RPMDB_BACKEND=$(rpm -E "%{_db_backend}")
+PARENT_DB_PATH=$(dirname "${RPMDB_PATH}")
+
+echo "using ${RPMDB_BACKEND} backend"
+
+# If correct rpmdb file for current backend does not exist, try to rebuild rpmdb to add the correct files
+if [ ! -e "${RPMDB_PATH}/${RPMDB_FILES[${RPMDB_BACKEND}]}" ]; then
+        echo "${RPMDB_FILES[${RPMDB_BACKEND}]} does not exist"
+        echo "Rebuilding rpmdb..."
+        if ! rpmdb --rebuilddb; then
+                echo "Rebuild of rpmdb failed. Trying to rebuild manually..."
+                REBUILD_FOLDER="$(ls $(dirname "${RPMDB_PATH}") | grep -m 1 rpmrebuild)"
+                REBUILD_PATH="${PARENT_DB_PATH}/${REBUILD_FOLDER}"
+                rm -fr "${RPMDB_PATH}"/*
+                cp -a "${REBUILD_PATH}"/* "${RPMDB_PATH}"/
+                rm -rf "${REBUILD_PATH}"
+        fi
+
+        echo "Rebuild of rpmdb done"
+fi
+
+# Simple rpmdb verification
+if [ "${RPMDB_BACKEND}" == "sqlite" ]; then
+    sqlite3 "${RPMDB_PATH}"/rpmdb.sqlite "pragma integrity_check;"
+fi
+
+if [ "${RPMDB_BACKEND}" == "bdb" ]; then
+    /usr/lib/rpm/rpmdb_verify "${RPMDB_PATH}"/Packages
+fi
+
+rpmdb -vv --verifydb
+
+echo "DONE"
+
+exit 0

From 54068c5c8f018e85d8bd48d2eedcd279a3c61969 Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <oscar.dominguez.ortiz@intel.com>
Date: Jul 04 2022 22:01:27 +0000
Subject: [PATCH 2/2] Change rpmdb verify


---

diff --git a/Containerfile.c8s b/Containerfile.c8s
index 71a1634..bcbdf95 100644
--- a/Containerfile.c8s
+++ b/Containerfile.c8s
@@ -15,7 +15,7 @@ ENV DATE="08:00 next Fri"
 RUN adduser build
 
 RUN dnf -y update
-RUN dnf -y install vi buildah podman
+RUN dnf -y install vi buildah
 RUN dnf -y clean all && \
     rm -rf /var/cache /var/log/dnf*
 
@@ -46,8 +46,10 @@ RUN mkdir -p /var/lib/shared/vfs-images \
 
 COPY make-hyperscale-container.sh /home/build
 COPY runner.sh /home/build
+COPY validate/verify_rpmdb.sh /home/build
 RUN chmod a+x /home/build/make-hyperscale-container.sh
 RUN chmod a+x /home/build/runner.sh
+RUN chmod a+x /home/build/verify_rpmdb.sh
 
 RUN chown build:build -R /home/build
 
diff --git a/Containerfile.c9s b/Containerfile.c9s
index ae5d5a2..3533bb8 100644
--- a/Containerfile.c9s
+++ b/Containerfile.c9s
@@ -15,7 +15,7 @@ ENV DATE="08:00 next Fri"
 RUN adduser build
 
 RUN dnf -y update
-RUN dnf -y install vi buildah
+RUN dnf -y install vi buildah sqlite
 RUN dnf -y clean all && \
     rm -rf /var/cache /var/log/dnf*
 
@@ -46,8 +46,10 @@ RUN mkdir -p /var/lib/shared/vfs-images \
 
 COPY make-hyperscale-container.sh /home/build
 COPY runner.sh /home/build
+COPY validate/verify_rpmdb.sh /home/build
 RUN chmod a+x /home/build/make-hyperscale-container.sh
 RUN chmod a+x /home/build/runner.sh
+RUN chmod a+x /home/build/verify_rpmdb.sh
 
 RUN chown build:build -R /home/build
 
diff --git a/runner.sh b/runner.sh
index fb5e2bb..11c9428 100644
--- a/runner.sh
+++ b/runner.sh
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -eu
+
 export TZ='America/Los_Angeles'
 
 # Hyperscale Centos Stream version, default 8
@@ -18,6 +20,7 @@ Push () {
 
 # Always run once immediately for easy testing.
 ./make-hyperscale-container.sh $VERSION
+./verify_rpmdb.sh
 Push
 
 while true; do
@@ -26,6 +29,7 @@ while true; do
   sleep "$TIME"
   echo "Starting at $(date)"
   ./make-hyperscale-container.sh $VERSION
+  ./verify_rpmdb.sh
   Push
   echo "Done at $(date)"
   echo
diff --git a/test_rpmdb.sh b/test_rpmdb.sh
deleted file mode 100644
index 4814116..0000000
--- a/test_rpmdb.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-# rpmdb files for each rpmdb backend
-declare -A RPMDB_FILES
-RPMDB_FILES[ndb]="Packages.db"
-RPMDB_FILES[sqlite]="rpmdb.sqlite"
-RPMDB_FILES[bdb]="Packages"
-
-RPMDB_PATH=$(rpm -E "%{_dbpath}")
-RPMDB_BACKEND=$(rpm -E "%{_db_backend}")
-PARENT_DB_PATH=$(dirname "${RPMDB_PATH}")
-
-echo "using ${RPMDB_BACKEND} backend"
-
-# If correct rpmdb file for current backend does not exist, try to rebuild rpmdb to add the correct files
-if [ ! -e "${RPMDB_PATH}/${RPMDB_FILES[${RPMDB_BACKEND}]}" ]; then
-        echo "${RPMDB_FILES[${RPMDB_BACKEND}]} does not exist"
-        echo "Rebuilding rpmdb..."
-        if ! rpmdb --rebuilddb; then
-                echo "Rebuild of rpmdb failed. Trying to rebuild manually..."
-                REBUILD_FOLDER="$(ls $(dirname "${RPMDB_PATH}") | grep -m 1 rpmrebuild)"
-                REBUILD_PATH="${PARENT_DB_PATH}/${REBUILD_FOLDER}"
-                rm -fr "${RPMDB_PATH}"/*
-                cp -a "${REBUILD_PATH}"/* "${RPMDB_PATH}"/
-                rm -rf "${REBUILD_PATH}"
-        fi
-
-        echo "Rebuild of rpmdb done"
-fi
-
-# Simple rpmdb verification
-if [ "${RPMDB_BACKEND}" == "sqlite" ]; then
-    sqlite3 "${RPMDB_PATH}"/rpmdb.sqlite "pragma integrity_check;"
-fi
-
-if [ "${RPMDB_BACKEND}" == "bdb" ]; then
-    /usr/lib/rpm/rpmdb_verify "${RPMDB_PATH}"/Packages
-fi
-
-rpmdb -vv --verifydb
-
-echo "DONE"
-
-exit 0
diff --git a/validate/verify_rpmdb.sh b/validate/verify_rpmdb.sh
new file mode 100644
index 0000000..1dfa4bb
--- /dev/null
+++ b/validate/verify_rpmdb.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -eux
+
+# start container for testing
+ctr=$(buildah from "centos-stream-hyperscale-${VERSION}")
+
+RPMDB_PATH=$(buildah run $ctr -- rpm -E "%{_dbpath}")
+RPMDB_BACKEND=$(buildah run $ctr -- rpm -E "%{_db_backend}")
+
+# Check if the container is using sqlite or bdb backend
+if [ "${RPMDB_BACKEND}" != "sqlite" ] && [ "${RPMDB_BACKEND}" != "bdb" ]; then
+       echo "script only works with sqlite or bdb rpmdb backend"
+       buildah rm $ctr
+       exit 1
+fi
+
+# Simple rpmdb verification
+# for c9s mount container to avoid install sqlite every time
+if [ "${RPMDB_BACKEND}" == "sqlite" ]; then
+  mnt=$(buildah unshare -- sh -c "buildah mount $ctr")
+  sqlite3 "$mnt${RPMDB_PATH}"/rpmdb.sqlite "pragma integrity_check;"
+  buildah unshare -- sh -c "buildah unmount $ctr"
+else
+    buildah run $ctr -- /usr/lib/rpm/rpmdb_verify "${RPMDB_PATH}"/Packages
+fi
+
+buildah run $ctr -- rpmdb -vv --verifydb
+
+# Query all headers in the DB
+buildah run $ctr -- rpm -qa 1> /dev/null
+
+# Remove container
+buildah rm $ctr
+
+exit 0