From 8acdab3122e5101ffc0d2cccc988e9ac7faaf9fe Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Aug 09 2024 14:00:12 +0000
Subject: composes: keep metadata where it won't be garbage collected


This copies the metadata for nightly composes to a separate
location which will not be garbage-collected, so we have a store
of metadata from old composes now PDC is gone. It also writes
the compose's compose ID into a "series" file, which makes it
possible to sanely do things like "what was the compose before
this compose?" without having PDC.

The metadata for a full compose weighs about 150M. Almost all of
this is rpms.json; withut that it's under 100K. If 150M per
compose for eternity is too big, we could leave out rpms.json.

The series file for Rawhide will grow eternally, but it's just a
text file. We do one compose per day (usually), so in 100 years
it'll be 36500 lines long, which doesn't really seem like a
problem.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

---

diff --git a/cloud-nightly.sh b/cloud-nightly.sh
index 13ef767..0970ab9 100755
--- a/cloud-nightly.sh
+++ b/cloud-nightly.sh
@@ -7,6 +7,8 @@ LABEL=$1
 shift
 CONFIG="fedora-cloud.conf"
 TARGET_DIR="/mnt/koji/compose/cloud"
+MD_ARCHIVE_DIR="/mnt/koji/compose/metadata-archive"
+SERIES_DIR="${MD_ARCHIVE_DIR}/series"
 #OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
 NIGHTLY=""
 DEST=$(pwd)
@@ -79,4 +81,11 @@ SUBJECT='Fedora '$RELEASE' compose report: '$SHORTCOMPOSE_ID' changes'
 #         mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail
 #done
 
+# Copy the metadata to the non-garbage-collected metadata archive
+mkdir -p "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}"
+cp "${DESTDIR}/compose/metadata/*.json" "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}/"
+# Add the compose ID to the 'series' file which records the
+# compose IDs for each SHORT name in order
+echo "${NEWCOMPOSE_ID}" >> "${SERIES_DIR}/${SHORT}-${RELEASE}"
+
 find $TARGET_DIR  -xdev -depth -maxdepth 2 -mtime +14  -exec rm -rf {} \;
diff --git a/container-nightly.sh b/container-nightly.sh
index 7b4994f..c5bc42e 100755
--- a/container-nightly.sh
+++ b/container-nightly.sh
@@ -6,6 +6,8 @@ LABEL=$1
 shift
 CONFIG="fedora-container.conf"
 TARGET_DIR="/mnt/koji/compose/container"
+MD_ARCHIVE_DIR="/mnt/koji/compose/metadata-archive"
+SERIES_DIR="${MD_ARCHIVE_DIR}/series"
 #OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
 NIGHTLY=""
 DEST=$(pwd)
@@ -87,4 +89,11 @@ pushd ./scripts
 popd
 popd
 
+# Copy the metadata to the non-garbage-collected metadata archive
+mkdir -p "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}"
+cp "${DESTDIR}/compose/metadata/*.json" "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}/"
+# Add the compose ID to the 'series' file which records the
+# compose IDs for each SHORT name in order
+echo "${NEWCOMPOSE_ID}" >> "${SERIES_DIR}/${SHORT}-${RELEASE}"
+
 find $TARGET_DIR  -xdev -depth -maxdepth 2 -mtime +14  -exec rm -rf {} \;
diff --git a/nightly.sh b/nightly.sh
index 40c174a..1d733ea 100755
--- a/nightly.sh
+++ b/nightly.sh
@@ -5,6 +5,8 @@ export LC_ALL=C
 
 CONFIG="fedora.conf"
 TARGET_DIR="/mnt/koji/compose/rawhide"
+MD_ARCHIVE_DIR="/mnt/koji/compose/metadata-archive"
+SERIES_DIR="$MD_ARCHIVE_DIR/series"
 #OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=$TARGET_DIR"
 NIGHTLY="--nightly"
 SKIP_PHASES=""
@@ -237,6 +239,13 @@ for tomail in $TOMAIL ; do
          mutt -e "set from=\"$FROM\"" -e 'set envelope_from=yes' -s "$SUBJECT" $tomail
 done
 
+# Copy the metadata to the non-garbage-collected metadata archive
+mkdir -p "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}"
+cp "${DESTDIR}/compose/metadata/*.json" "${MD_ARCHIVE_DIR}/${NEWCOMPOSE_ID}/"
+# Add the compose ID to the 'series' file which records the
+# compose IDs for each SHORT name in order
+echo "${NEWCOMPOSE_ID}" >> "${SERIES_DIR}/${SHORT}-${RELEASE}"
+
 # Removed all the older than 14 days composes
 find $TARGET_DIR  -xdev -depth -maxdepth 2 -mtime +14  -exec rm -rf {} \;
 send_fedora_message "${fedora_message_json_done}" ${RELEASE} cleanup.complete
diff --git a/release-candidate.sh b/release-candidate.sh
index ac87f99..684c0aa 100755
--- a/release-candidate.sh
+++ b/release-candidate.sh
@@ -31,6 +31,8 @@ fi
 # Extract the Fedora version from the label. Example label: 39_RC-1.1
 FEDORA_VERSION="${LABEL%%_*}"
 TARGET_DIR="/mnt/koji/compose/${FEDORA_VERSION}"
+MD_ARCHIVE_DIR="/mnt/koji/compose/metadata-archive"
+SERIES_DIR="${MD_ARCHIVE_DIR}/series"
 # OLD_COMPOSES_DIR="--old-composes=/mnt/fedora_koji/compose/f23 --old-composes=${TARGET_DIR}"
 NIGHTLY=""
 SKIP_PHASES="--skip-phase=productimg"
@@ -211,6 +213,13 @@ if [ -d "${RSYNCTARGET}" ]; then
 fi
 echo ""
 
+# Copy the metadata to the non-garbage-collected metadata archive
+mkdir -p "${MD_ARCHIVE_DIR}/${COMPOSE_ID_2}"
+cp "${DESTDIR}/compose/metadata/*.json" "${MD_ARCHIVE_DIR}/${COMPOSE_ID_2}/"
+# Add the compose ID to the 'series' file which records the
+# compose IDs for each SHORT name in order
+echo "${COMPOSE_ID_2}" >> "${SERIES_DIR}/${SHORT}-${RELEASE}"
+
 echo "========================================"
 echo "LET INTERESTED PERSONS KNOW THAT THE RSYNC IS COMPLETE."
 echo ""