From a87bda96b8d8f1500663b636539f2c17c8378f50 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Apr 20 2018 12:14:29 +0000 Subject: Allow setting dry run mode per event. --- diff --git a/freshmaker/events.py b/freshmaker/events.py index 3a4b5e8..3214e61 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -35,15 +35,17 @@ class BaseEvent(object): _parsers = {} - def __init__(self, msg_id, manual=False): + def __init__(self, msg_id, manual=False, dry_run=False): """ A base class to abstract events from different fedmsg messages. :param msg_id: the id of the msg (e.g. 2016-SomeGUID) :param manual: True if the event was trigerred manually by Freshmaker REST API. + :param dry_run: True if the event should be handled in DRY_RUN mode. """ self.msg_id = msg_id self.manual = manual + self.dry_run = False # Moksha calls `consumer.validate` on messages that it receives, and # even though we have validation turned off in the config there's still @@ -320,6 +322,7 @@ class ODCSComposeStateChangeEvent(BaseEvent): class FreshmakerManualRebuildEvent(BaseEvent): - def __init__(self, msg_id, errata_id=None): - super(FreshmakerManualRebuildEvent, self).__init__(msg_id) + def __init__(self, msg_id, errata_id=None, dry_run=False): + super(FreshmakerManualRebuildEvent, self).__init__( + msg_id, dry_run=dry_run) self.errata_id = errata_id diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index aaaf692..028660c 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -115,6 +115,7 @@ class BaseHandler(object): self._db_event_id = None self._db_artifact_build_id = None self._log_prefix = "" + self._force_dry_run = False def _log(self, log_fnc, msg, *args, **kwargs): """ @@ -152,6 +153,21 @@ class BaseHandler(object): """ return self._log(log.error, msg, *args, **kwargs) + def force_dry_run(self): + """ + Forces the handling of the current even in DRY_RUN mode. + """ + self._force_dry_run = True + + @property + def dry_run(self): + """ + Returns True if the event should be hanled in DRY_RUN mode. + """ + if self._force_dry_run: + return True + return conf.dry_run + @property def current_db_event_id(self): return self._db_event_id @@ -387,7 +403,9 @@ class ContainerBuildHandler(BaseHandler): :return: task id returned from Koji buildContainer API. :rtype: int """ - with koji_service(profile=conf.koji_profile, logger=log) as service: + with koji_service( + profile=conf.koji_profile, logger=log, + dry_run=self.dry_run) as service: log.info('Building container from source: %s, ' 'release=%r, parent=%r, target=%r', scm_url, release, koji_parent_build, target) @@ -451,7 +469,7 @@ class ContainerBuildHandler(BaseHandler): `compose_id`. In DRY_RUN mode, returns fake compose information without contacting the ODCS server. """ - if conf.dry_run: + if self.dry_run: compose = {} compose['id'] = compose_id compose['result_repofile'] = "http://localhost/%d.repo" % ( diff --git a/freshmaker/handlers/bodhi/update_complete_stable.py b/freshmaker/handlers/bodhi/update_complete_stable.py index 0cac3f6..b7e96b3 100644 --- a/freshmaker/handlers/bodhi/update_complete_stable.py +++ b/freshmaker/handlers/bodhi/update_complete_stable.py @@ -76,7 +76,8 @@ class BodhiUpdateCompleteStableHandler(BaseHandler): def get_rpms_included_in_bodhi_update(self, builds): build_nvrs = (build['nvr'] for build in builds) - with koji_service(profile=conf.koji_profile, logger=log) as service: + with koji_service(profile=conf.koji_profile, logger=log, + dry_run=conf.dry_run) as service: return chain(*[service.get_build_rpms(nvr) for nvr in build_nvrs]) def get_containers_including_rpms(self, rpms): diff --git a/freshmaker/handlers/brew/container_task_state_change.py b/freshmaker/handlers/brew/container_task_state_change.py index 70cd073..2dda040 100644 --- a/freshmaker/handlers/brew/container_task_state_change.py +++ b/freshmaker/handlers/brew/container_task_state_change.py @@ -48,6 +48,8 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): When build container task state changed in brew, update build state in db and rebuild containers depend on the success build as necessary. """ + if event.dry_run: + self.force_dry_run() build_id = event.task_id @@ -66,7 +68,9 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): if found_build.event.event_type_id == EVENT_TYPES[ErrataAdvisoryRPMsSignedEvent]: errata_id = found_build.event.search_key # build_id is actually task id in build system, find out the actual build first - with koji_service(conf.koji_profile, log, login=False) as session: + with koji_service( + conf.koji_profile, log, login=False, + dry_run=self.dry_run) as session: container_build_id = session.get_container_build_id_from_task(build_id) ret, msg = self._verify_advisory_rpms_in_container_build(errata_id, container_build_id) @@ -137,7 +141,7 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): """ verify container built on brew has the latest rpms from an advisory """ - if conf.dry_run: + if self.dry_run: return (True, '') # Get rpms in advisory. There can be multiple versions of RPMs with @@ -147,7 +151,9 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): e = Errata() build_nvrs = e.get_builds(errata_id) if build_nvrs: - with koji_service(conf.koji_profile, log, login=False) as session: + with koji_service( + conf.koji_profile, log, login=False, + dry_run=self.dry_run) as session: for build_nvr in build_nvrs: build_rpms = session.get_build_rpms(build_nvr) for rpm in build_rpms: @@ -156,7 +162,9 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): advisory_rpms_by_name[rpm['name']].add(rpm['nvr']) # get rpms in container - with koji_service(conf.koji_profile, log, login=False) as session: + with koji_service( + conf.koji_profile, log, login=False, + dry_run=self.dry_run) as session: container_rpms = session.get_rpms_in_container(container_build_id) container_rpms_by_name = { rpmlib.parse_nvr(x)['name']: x for x in container_rpms} diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index bf641c6..fb32cad 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -70,10 +70,13 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): advisory. """ + if event.dry_run: + self.force_dry_run() + # In case we run in DRY_RUN mode, we need to initialize # FAKE_COMPOSE_ID to the id of last ODCS compose to give the IDs # increasing and unique even between Freshmaker restarts. - if conf.dry_run: + if self.dry_run: ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID = \ Compose.get_highest_compose_id(db.session) + 1 @@ -94,7 +97,8 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): advisory_name=event.advisory.name, advisory_security_impact=event.advisory.security_impact, advisory_highest_cve_severity=event.advisory.highest_cve_severity, - advisory_product_short_name=event.advisory.product_short_name): + advisory_product_short_name=event.advisory.product_short_name, + dry_run=self.dry_run): msg = ("Errata advisory {0} is not allowed by internal policy " "to trigger rebuilds.".format(event.advisory.errata_id)) db_event.transition(EventState.SKIPPED, msg) @@ -217,6 +221,7 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): # Generate and inject the ODCSComposeStateChangeEvent event. event = ODCSComposeStateChangeEvent( "fake_compose_msg", new_compose) + event.dry_run = True self.log_info("Injecting fake event: %r", event) work_queue_put(event) @@ -290,7 +295,7 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): 'source: %s, source type: %s, packages: %s', compose_source, 'tag', packages) - if not conf.dry_run: + if not self.dry_run: with krb_context(): new_compose = create_odcs_client().new_compose( compose_source, 'tag', packages=packages, @@ -317,7 +322,7 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): content_sets) odcs = create_odcs_client() - if not conf.dry_run: + if not self.dry_run: with krb_context(): new_compose = odcs.new_compose( ' '.join(content_sets), 'pulp') @@ -380,7 +385,8 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): return None def _get_base_image_build_tag(self, build_target): - with koji_service(conf.koji_profile, log) as session: + with koji_service( + conf.koji_profile, log, dry_run=self.dry_run) as session: target_info = session.get_build_target(build_target) if target_info is None: return target_info @@ -396,7 +402,7 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): if not build_tag: return None - if conf.dry_run: + if self.dry_run: new_compose = self._fake_odcs_new_compose( build_tag, 'tag', results=['boot.iso']) else: @@ -412,7 +418,8 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): :return: list of RPM names built from given build. :rtype: list """ - with koji_service(conf.koji_profile, log) as session: + with koji_service( + conf.koji_profile, log, dry_run=self.dry_run) as session: rpms = session.get_build_rpms(nvr) return list(set([rpm['name'] for rpm in rpms])) @@ -423,7 +430,8 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): of found tag. :rtype: str """ - with koji_service(conf.koji_profile, log) as service: + with koji_service( + conf.koji_profile, log, dry_run=self.dry_run) as service: # Get the list of *-candidate tags, because packages added into # Errata should be tagged into -candidate tag. tags = service.session.listTags(nvr) @@ -703,7 +711,8 @@ class ErrataAdvisoryRPMsSignedHandler(ContainerBuildHandler): def _find_build_srpm_name(self, build_nvr): """Find srpm name from a build""" - with koji_service(conf.koji_profile, log) as session: + with koji_service( + conf.koji_profile, log, dry_run=self.dry_run) as session: rpm_infos = session.get_build_rpms(build_nvr, arches='src') if not rpm_infos: raise ValueError( diff --git a/freshmaker/handlers/internal/manual_rebuild.py b/freshmaker/handlers/internal/manual_rebuild.py index 24a31ab..230f458 100644 --- a/freshmaker/handlers/internal/manual_rebuild.py +++ b/freshmaker/handlers/internal/manual_rebuild.py @@ -66,6 +66,7 @@ class FreshmakerManualRebuildHandler(ContainerBuildHandler): new_event = ErrataAdvisoryStateChangedEvent( manual_rebuild_event.msg_id + "." + str(advisory.name), advisory) new_event.manual = True + new_event.dry_run = manual_rebuild_event.dry_run msg = ("Generated ErrataAdvisoryStateChangedEvent (%s) for errata: %s" % (manual_rebuild_event.msg_id, manual_rebuild_event.errata_id)) self.current_db_event.transition(EventState.COMPLETE, msg) diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index b4463a5..215234e 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -52,8 +52,9 @@ class KojiService(object): # Used to generate incremental task id in dry run mode. _FAKE_TASK_ID = 1 - def __init__(self, profile=None): + def __init__(self, profile=None, dry_run=False): self._config = koji.read_config(profile or 'koji') + self.dry_run = dry_run @property def config(self): @@ -80,7 +81,7 @@ class KojiService(object): def krb_login(self): # No need to login on dry run, this makes dry run much faster. - if not conf.dry_run: + if not self.dry_run: self.session.krb_login(principal=conf.krb_auth_principal, keytab=conf.krb_auth_client_keytab) else: @@ -117,6 +118,7 @@ class KojiService(object): event = BrewContainerTaskStateChangeEvent( "fake_koji_msg_%d" % task_id, container, build_opts["git_branch"], build_target, task_id, "BUILDING", "CLOSED") + event.dry_run = self.dry_run # Inject the fake event. log.info("DRY RUN: Injecting fake event: %r", event) @@ -147,7 +149,7 @@ class KojiService(object): log.debug('Build from target: %s', build_target) log.debug('Build options: %s', build_opts) - if not conf.dry_run: + if not self.dry_run: task_id = self.session.buildContainer(source_url, build_target, build_opts) else: @@ -189,7 +191,7 @@ class KojiService(object): task result. If not found, return None. """ # We cannot get the build_id from task_id in dry_run mode... - if conf.dry_run: + if self.dry_run: return None build_id = None @@ -265,7 +267,7 @@ class KojiService(object): @contextlib.contextmanager -def koji_service(profile=None, logger=None, login=True): +def koji_service(profile=None, logger=None, login=True, dry_run=False): """A Koji service context manager that could be used with with Example:: @@ -281,7 +283,7 @@ def koji_service(profile=None, logger=None, login=True): with KojiService(koji='stg', logger=logger) as service: ... """ - service = KojiService(profile=profile) + service = KojiService(profile=profile, dry_run=dry_run) if login: if not conf.krb_auth_principal: @@ -294,7 +296,7 @@ def koji_service(profile=None, logger=None, login=True): service.krb_login() # We are not logged in in dry run mode... - if not conf.dry_run and not service.logged_in: + if not dry_run and not service.logged_in: log.error('Could not login server %s', service.server) yield None diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index ec57dc4..e9457f5 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -166,7 +166,8 @@ class ContainerImage(dict): """ data = self._get_default_additional_data() - with koji_service(conf.koji_profile, log) as session: + with koji_service( + conf.koji_profile, log, dry_run=conf.dry_run) as session: build = session.get_build(nvr) if not build: raise KojiLookupError( diff --git a/freshmaker/parsers/internal/manual_rebuild.py b/freshmaker/parsers/internal/manual_rebuild.py index c698fd8..6b95bb5 100644 --- a/freshmaker/parsers/internal/manual_rebuild.py +++ b/freshmaker/parsers/internal/manual_rebuild.py @@ -36,5 +36,7 @@ class FreshmakerManualRebuildParser(BaseParser): msg_id = msg.get('msg_id') inner_msg = msg.get('msg') errata_id = inner_msg.get('errata_id') + dry_run = inner_msg.get('dry_run', False) - return FreshmakerManualRebuildEvent(msg_id, errata_id=errata_id) + return FreshmakerManualRebuildEvent( + msg_id, errata_id=errata_id, dry_run=dry_run) diff --git a/tests/test_handler.py b/tests/test_handler.py index bebef81..4c6e073 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -131,6 +131,16 @@ class TestContext(helpers.ModelsTestCase): self.assertRaises(ProgrammingError, handler.set_context, "something") +class TestDryRun(helpers.FreshmakerTestCase): + + def test_force_dry_run(self): + handler = MyHandler() + self.assertFalse(handler.dry_run) + + handler.force_dry_run() + self.assertTrue(handler.dry_run) + + class TestGetRepoURLs(helpers.ModelsTestCase): def setUp(self):