From 2a1e4236376cd034edc7cacf8497e72186360cf1 Mon Sep 17 00:00:00 2001 From: Joe Talbott Date: Nov 11 2021 20:13:35 +0000 Subject: [PATCH 1/5] Prevent kojira from attempting to remove repos on other volumes. * Adds '--ignore-other-volumes' flag and configuration option. --- diff --git a/util/kojira b/util/kojira index 445cacb..fcc6390 100755 --- a/util/kojira +++ b/util/kojira @@ -445,6 +445,15 @@ class RepoManager(object): self.logger.info('Found repo %s, state=%s' % (repo_id, koji.REPO_STATES[data['state']])) repo = ManagedRepo(self, data, repodata) + if self.options.ignore_other_volumes: + info = repo.get_info() + volume = info.get('volume') + if volume is not None and volume != 'DEFAULT': + # Other volume + self.logger.info("Skipping repo ({}) on other volume {}".format( + repo_id, volume, + )) + continue self.repos[repo_id] = repo if not getTag(self.session, repo.tag_id) and not repo.expired(): self.logger.info('Tag %d for repo %d disappeared, expiring.', repo.tag_id, repo_id) @@ -635,9 +644,11 @@ class RepoManager(object): def pruneLocalRepos(self): for volinfo in self.session.listVolumes(): - volumedir = pathinfo.volumedir(volinfo['name']) - repodir = "%s/repos" % volumedir - self._pruneLocalRepos(repodir, self.options.deleted_repo_lifetime) + volname = volinfo['name'] + volumedir = pathinfo.volumedir(volname) + if volname == 'DEFAULT': # currently the only path for non-dist repos. + repodir = "%s/repos" % volumedir + self._pruneLocalRepos(repodir, self.options.deleted_repo_lifetime) distrepodir = "%s/repos-dist" % volumedir self._pruneLocalRepos(distrepodir, self.options.dist_repo_lifetime) @@ -1198,6 +1209,8 @@ def get_options(): parser.add_option("--logfile", help="Specify logfile") parser.add_option("--queue-file", help="If specified, queue is dumped to separate status file each cycle") + parser.add_option("--ignore-other-volumes", action="store_true", + help="Ignore repos on other volumes") (options, args) = parser.parse_args() config = koji.read_config_files(options.configFile) @@ -1235,6 +1248,7 @@ def get_options(): 'cert': None, 'serverca': None, 'queue_file': None, + 'ignore_other_volumes': False, } if config.has_section(section): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', @@ -1245,7 +1259,7 @@ def get_options(): 'cert', 'serverca', 'debuginfo_tags', 'queue_file', 'source_tags', 'separate_source_tags', 'ignore_tags') bool_opts = ('verbose', 'debug', 'ignore_stray_repos', 'offline_retry', - 'no_ssl_verify', 'check_external_repos') + 'no_ssl_verify', 'check_external_repos', 'ignore_other_volumes') legacy_opts = ('with_src', 'delete_batch_size', 'recent_tasks_lifetime') for name in config.options(section): if name in int_opts: From bf3404eece0cf6ccacef13d733d2d843f30bab60 Mon Sep 17 00:00:00 2001 From: Joe Simmons-Talbott Date: Nov 11 2021 20:13:35 +0000 Subject: [PATCH 2/5] kojira: make 'ignore_other_volumes' a config only option. * pull 'DEFAULT' repo handling out of the loop. * Add 'ignore_other_volumes' to kojira.conf --- diff --git a/util/kojira b/util/kojira index fcc6390..246d5ae 100755 --- a/util/kojira +++ b/util/kojira @@ -643,12 +643,14 @@ class RepoManager(object): session.logout() def pruneLocalRepos(self): + volname = 'DEFAULT' + volumedir = pathinfo.volumedir(volname) + repodir = "%s/repos" % volumedir + self._pruneLocalRepos(repodir, self.options.deleted_repo_lifetime) + for volinfo in self.session.listVolumes(): volname = volinfo['name'] volumedir = pathinfo.volumedir(volname) - if volname == 'DEFAULT': # currently the only path for non-dist repos. - repodir = "%s/repos" % volumedir - self._pruneLocalRepos(repodir, self.options.deleted_repo_lifetime) distrepodir = "%s/repos-dist" % volumedir self._pruneLocalRepos(distrepodir, self.options.dist_repo_lifetime) @@ -1209,8 +1211,6 @@ def get_options(): parser.add_option("--logfile", help="Specify logfile") parser.add_option("--queue-file", help="If specified, queue is dumped to separate status file each cycle") - parser.add_option("--ignore-other-volumes", action="store_true", - help="Ignore repos on other volumes") (options, args) = parser.parse_args() config = koji.read_config_files(options.configFile) diff --git a/util/kojira.conf b/util/kojira.conf index e3bb2d0..31160fc 100644 --- a/util/kojira.conf +++ b/util/kojira.conf @@ -46,3 +46,6 @@ logfile=/var/log/kojira.log ; as otherwise you can end with weird behaviour. For details see ; https://pagure.io/koji/issue/2159 ; check_external_repos = false + +; don't attempt to remove repos on non-default volumes +; ignore_other_volumes = false From 60bf4d128265e5f5b9fc2bc374e5493dd8478da0 Mon Sep 17 00:00:00 2001 From: Joe Simmons-Talbott Date: Nov 11 2021 20:13:35 +0000 Subject: [PATCH 3/5] Fix for case where 'info' is None. --- diff --git a/util/kojira b/util/kojira index 246d5ae..5a2b530 100755 --- a/util/kojira +++ b/util/kojira @@ -447,6 +447,8 @@ class RepoManager(object): repo = ManagedRepo(self, data, repodata) if self.options.ignore_other_volumes: info = repo.get_info() + if info is None: + continue volume = info.get('volume') if volume is not None and volume != 'DEFAULT': # Other volume From c82c991d3f7ecc802f6bd837d9440cf6fcb17cca Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Nov 11 2021 20:13:35 +0000 Subject: [PATCH 4/5] we should still track repos that the hub tells us about --- diff --git a/util/kojira b/util/kojira index 5a2b530..5dd5091 100755 --- a/util/kojira +++ b/util/kojira @@ -445,17 +445,6 @@ class RepoManager(object): self.logger.info('Found repo %s, state=%s' % (repo_id, koji.REPO_STATES[data['state']])) repo = ManagedRepo(self, data, repodata) - if self.options.ignore_other_volumes: - info = repo.get_info() - if info is None: - continue - volume = info.get('volume') - if volume is not None and volume != 'DEFAULT': - # Other volume - self.logger.info("Skipping repo ({}) on other volume {}".format( - repo_id, volume, - )) - continue self.repos[repo_id] = repo if not getTag(self.session, repo.tag_id) and not repo.expired(): self.logger.info('Tag %d for repo %d disappeared, expiring.', repo.tag_id, repo_id) From 5e2685a424d1adaea2f7392b01315adb4dd60594 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Nov 11 2021 20:13:35 +0000 Subject: [PATCH 5/5] rework a bit --- diff --git a/util/kojira b/util/kojira index 5dd5091..49bbc90 100755 --- a/util/kojira +++ b/util/kojira @@ -256,6 +256,10 @@ class ManagedRepo(object): if not os.path.exists(realpath): logger.error('Repo real path missing: %s', realpath) return False + if self.options.ignore_other_volumes: + # don't delete from other volumes + logger.error('Repo on non-default volume %s', realpath) + return False if not os.path.samefile(path, realpath): logger.error('Incorrect volume link: %s', path) return False @@ -641,6 +645,11 @@ class RepoManager(object): for volinfo in self.session.listVolumes(): volname = volinfo['name'] + if volname == 'DEFAULT': + continue + if self.options.ignore_other_volumes: + # don't prune from other volumes + continue volumedir = pathinfo.volumedir(volname) distrepodir = "%s/repos-dist" % volumedir self._pruneLocalRepos(distrepodir, self.options.dist_repo_lifetime)