From d7002e6f883b0427ff6c9e3df932ddb06b2e90bd Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 15 2017 13:07:59 +0000 Subject: [PATCH 1/2] Restructure Docker image handling and support multiple advisories in a rebuild and multiple packages in single advisory. --- diff --git a/freshmaker/config.py b/freshmaker/config.py index 8277e16..88fe512 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -232,6 +232,10 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Server URL to ODCS'}, + 'odcs_verify_ssl': { + 'type': bool, + 'default': True, + 'desc': 'Whether to enable SSL verification over HTTP with ODCS.'}, } def __init__(self, conf_section_obj): diff --git a/freshmaker/errata.py b/freshmaker/errata.py index fe694de..a40513c 100644 --- a/freshmaker/errata.py +++ b/freshmaker/errata.py @@ -130,6 +130,17 @@ class Errata(object): return True + def get_builds(self, errata_id): + builds_per_product = self._errata_http_get( + "advisory/%s/builds.json" % str(errata_id)) + + # Store NVRs of all builds in advisory to nvrs set. + nvrs = set() + for builds in builds_per_product.values(): + for build in builds: + nvrs.update(set(build.keys())) + return nvrs + def get_pulp_repository_ids(self, errata_id): """Get Pulp repository IDs where packages included in errata will end up diff --git a/freshmaker/events.py b/freshmaker/events.py index aa8a55f..61741d5 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -248,6 +248,29 @@ class KojiTaskStateChangeEvent(BaseEvent): self.task_id = task_id self.task_state = task_state +class ErrataAdvisoryStateChangedEvent(BaseEvent): + """ + Represents change od Errata Advisory status. + """ + + def __init__(self, msg_id, errata_id, state): + super(ErrataAdvisoryStateChangedEvent, self).__init__(msg_id) + self.errata_id = errata_id + self.state = state + +class ErrataAdvisoryRPMsSignedEvent(BaseEvent): + """ + Event when all RPMs in Errata advisory are signed. + """ + def __init__(self, msg_id, errata_name, errata_id, security_impact): + super(ErrataAdvisoryRPMsSignedEvent, self).__init__(msg_id) + self.errata_name = errata_name + self.errata_id = errata_id + self.security_impact = security_impact + + @property + def search_key(self): + return str(self.errata_id) class BrewSignRPMEvent(BaseEvent): """ diff --git a/freshmaker/handlers/brew/sign_rpm.py b/freshmaker/handlers/brew/sign_rpm.py index eac8070..ce6fb89 100644 --- a/freshmaker/handlers/brew/sign_rpm.py +++ b/freshmaker/handlers/brew/sign_rpm.py @@ -30,20 +30,25 @@ from itertools import chain from freshmaker import conf from freshmaker import log from freshmaker import db -from freshmaker.events import BrewSignRPMEvent +from freshmaker.events import BrewSignRPMEvent, ErrataAdvisoryRPMsSignedEvent from freshmaker.handlers import BaseHandler from freshmaker.kojiservice import koji_service from freshmaker.lightblue import LightBlue from freshmaker.pulp import Pulp from freshmaker.errata import Errata from freshmaker.types import ArtifactType, ArtifactBuildState +from freshmaker.models import Event from odcs.client.odcs import ODCS from odcs.client.odcs import AuthMech class BrewSignRPMHandler(BaseHandler): - """Rebuild docker images when a RPM is signed in Brew""" + """ + Checks whether all RPMs in Errata advisories for signed package are signed + and in case they are, generates ErrataAdvisoryRPMsSignedEvent events for + each advisory. + """ name = 'BrewSignRPMHandler' @@ -51,156 +56,12 @@ class BrewSignRPMHandler(BaseHandler): return isinstance(event, BrewSignRPMEvent) def handle(self, event): - """Rebuild docker images which contains this signed RPM - - Before rebuilding docker images, freshmaker has to find which docker - images includes the signed RPM. As of writing this feature, this - information is stored in LightBlue, and need to use content_sets to - find out those images. - - There are several external services taking part in the process of - rebuilding docker images. - - * Errata Tool: get which advisories contains the signed RPM, and Pulp - repositories the signed RPM will end up eventually when shipped. - * Pulp: query content set with repositories got from Errata Tool. - * LightBlue: this is where to query docker images that contains RPMs - from those content sets. - """ - - batches = self._find_images_to_rebuild(event) - - if not batches: - log.info('Not find docker images to rebuild.') - return [] - - self._log_batches(batches) - self._record_batches(batches, event) - - repo_url = self._prepare_yum_repo(event) # noqa - - return [] - - def _prepare_yum_repo(self, event): - """Prepare a yum repo for rebuild - - Run a compose in ODCS to contain required RPMs for rebuilding images - later. - """ - odcs = ODCS(conf.odcs_server_url, auth_mech=AuthMech.Kerberos) - packages = self._get_packages_for_compose(event.nvr) - compose_source = self._get_compose_source(event.nvr) - - if compose_source is None: - log.error('Build %s is not the latest build in its all tags.', - event.nvr) - return - - log.info('Generate new compose for rebuild: ' - 'source: %s, source type: %s, packages: %s', - compose_source, 'tag', packages) - - new_compose = odcs.new_compose(compose_source, - 'tag', - packages=packages) - compose_id = new_compose['id'] - compose_url = '{0}/odcs/{1}/composes/{2}'.format( - odcs.server_url.strip('/'), odcs.api_version, compose_id) - - log.info('Waiting for ODCS to finish the compose: %s', compose_url) - - while True: - time.sleep(1) - - compose = odcs.get_compose(compose_id) - state = compose['state'] - if state == 0: # waiting for generating compose - log.info('Waiting for generating new compose') - elif state == 1: # generating in progress - log.info('ODCS is generating the compose') - elif state == 4: # Failed to generate compose - log.error('ODCS fails to generate compose: %s', compose_url) - log.error('Please consult ODCS to see what is wrong with it') - return - elif state == 2: # Succeed to generate compose - log.info('ODCS has finished to generate compose. Continue to rebuild') - break - else: - log.error('Got unexpected compose state {0} from ODCS.'.format(state)) - return - - log.info('Repo URL containing packages used to rebuild container: %s', - new_compose['result_repo']) - - return new_compose['result_repo'] - - def _get_packages_for_compose(self, nvr): - """Get RPMs of current build NVR""" - with koji_service(conf.koji_profile, log) as session: - rpms = session.get_build_rpms(nvr) - return list(set([rpm['name'] for rpm in rpms])) - - def _get_compose_source(self, nvr): - """Get tag from which to collect packages to compose""" - with koji_service(conf.koji_profile, log) as service: - tag = [tag['name'] for tag in service.session.listTags(nvr) - if tag['name'].endswith('-candidate')][0] - latest_build = service.session.listTagged( - tag, - latest=True, - package=koji.parse_NVR(nvr)['name']) - if latest_build and latest_build[0]['nvr'] == nvr: - return tag - - def _log_batches(self, batches): - """ - Logs the information about images to rebuilt using log.info(...). - :param batches list: Output of _find_images_to_rebuild(...). - """ - log.info('Found docker images to rebuild in following order:') - for i, batch in enumerate(batches): - log.info(' Batch %d (%d images):', i, len(batch)) - for image in batch: - based_on = "based on %s" % image["parent"]["brew"]["build"] \ - if image["parent"] else "base image" - log.info(' - %s#%s (%s)' % - (image["repository"], image["commit"], based_on)) - - def _record_batches(self, batches, event): - """ - Records the images from batches to database. - :param batches list: Output of _find_images_to_rebuild(...). - """ - - # Used as tmp dict with {brew_buil_id: ArtifactBuild, ...} mapping. - builds = {} - - for batch in batches: - for image in batch: - name = image["brew"]["build"] - parent_name = image["parent"]["brew"]["build"] \ - if image["parent"] else None - dep_on = builds[parent_name] if parent_name in builds else None - build = self.record_build( - event, name, ArtifactType.IMAGE, 0, dep_on, - ArtifactBuildState.PLANNED.value) - - build_args = {} - build_args["repository"] = image["repository"] - build_args["commit"] = image["commit"] - build_args["parent"] = parent_name - build.build_args = json.dumps(build_args) - db.session.commit() - - builds[name] = build - - def _find_images_to_rebuild(self, event): # When get a signed RPM, first step is to find out advisories - # containing that RPM and has to ensure all builds are signed. + # containing that RPM and ensure all builds are signed. errata = Errata(conf.errata_tool_server_url) advisories = errata.advisories_from_event(event) - # Filter out advisories which are not allow by configuration + # Filter out advisories which are not allowed by configuration. advisories = [advisory for advisory in advisories if self.allow_build( ArtifactType.IMAGE, advisory_name=advisory.name, @@ -216,36 +77,13 @@ class BrewSignRPMHandler(BaseHandler): 'docker image until signed.', advisories) return [] - # Use the advisories to find out Pulp repository IDs from Errata Tool - # and furthermore get content_sets from Pulp where signed RPM will end - # up eventually when advisories are shipped. - pulp_repo_ids = list(set(chain( - *[errata.get_pulp_repository_ids(advisory.errata_id) - for advisory in advisories] - ))) - - pulp = Pulp(server_url=conf.pulp_server_url, - username=conf.pulp_username, - password=conf.pulp_password) - content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids) - - log.info('RPM will end up within content sets %s', content_sets) - - # Query images from LightBlue by signed RPM's srpm name and found - # content sets - lb = LightBlue(server_url=conf.lightblue_server_url, - cert=conf.lightblue_certificate, - private_key=conf.lightblue_private_key) - - srpm_name = self._find_build_srpm_name(event.nvr) - return lb.find_images_to_rebuild(srpm_name, content_sets) - - def _find_build_srpm_name(self, build_nvr): - """Find srpm name from a build""" - with koji_service(conf.koji_profile, log) as session: - rpm_infos = session.get_build_rpms(build_nvr, arches='src') - if not rpm_infos: - raise ValueError( - 'Build {} does not have a SRPM, although this should not ' - 'happen in practice.'.format(build_nvr)) - return rpm_infos[0]['name'] + # Now we know that all advisories with this signed RPM have also other + # RPMs signed. We can then proceed and generate + # ErrataAdvisoryRPMsSignedEvent. + new_events = [] + for advisory in advisories: + new_event = ErrataAdvisoryRPMsSignedEvent( + event.msg_id + "." + str(advisory.name), advisory.name, + advisory.errata_id, advisory.security_impact) + new_events.append(new_event) + return new_events diff --git a/freshmaker/handlers/errata/__init__.py b/freshmaker/handlers/errata/__init__.py new file mode 100644 index 0000000..3b24429 --- /dev/null +++ b/freshmaker/handlers/errata/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .errata_advisory_rpms_signed import ErrataAdvisoryRPMsSignedHandler # noqa +from .errata_advisory_state_changed import ErrataAdvisoryStateChangedHandler # noqa diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py new file mode 100644 index 0000000..4242152 --- /dev/null +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi +# Written by Jan Kaluza + +import json +import koji +import time + +from itertools import chain + +from freshmaker import conf +from freshmaker import log +from freshmaker import db +from freshmaker.events import ErrataAdvisoryRPMsSignedEvent +from freshmaker.handlers import BaseHandler +from freshmaker.kojiservice import koji_service +from freshmaker.lightblue import LightBlue +from freshmaker.pulp import Pulp +from freshmaker.errata import Errata +from freshmaker.types import ArtifactType, ArtifactBuildState +from freshmaker.models import Event + +from odcs.client.odcs import ODCS +from odcs.client.odcs import AuthMech + + +class ErrataAdvisoryRPMsSignedHandler(BaseHandler): + """ + Rebuilds all Docker images which contain packages from the Errata + advisory. + """ + + name = 'ErrataAdvisoryRPMsSignedHandler' + + def can_handle(self, event): + return isinstance(event, ErrataAdvisoryRPMsSignedEvent) + + def handle(self, event): + """ + Rebuilds all Docker images which contain packages from the Errata + advisory. + """ + + # Check if we are allowed to build this advisory. + if not self.allow_build( + ArtifactType.IMAGE, advisory_name=event.errata_name, + advisory_security_impact=event.security_impact): + log.info("Errata advisory %s not allowed to trigger rebuilds.", + event.errata_name) + return [] + + # Generate the Database representation of `event`. + db_event = Event.get_or_create( + db.session, event.msg_id, event.search_key, event.__class__, + released=False) + db.session.commit() + + # Get and record all images to rebuild based on the current + # ErrataAdvisoryRPMsSignedEvent event. + builds = self._record_images_to_rebuild(db_event, event) + if not builds: + log.info('Not found docker images to rebuild.') + return [] + + # Generate the ODCS compose with RPMs from the current advisory. + repo_urls = [] + repo_urls.append(self._prepare_yum_repo(event)) # noqa + + # Find out extra events we want to include. These are advisories + # which are not released yet and touches some Docker images which + # are shared with the initial list of docker images we are going to + # rebuild. + # If we For example have NSS Errata advisory and httpd advisory, we + # need to rebuild some Docker images with both NSS and httpd + # advisories. + # We also want to search for extra events recursively, because there + # might for example be zlib advisory, and we want to include this zlib + # advisory when rebuilding NSS when rebuilding httpd... :) + prev_builds_count = 0 + seen_extra_events = [] + + # We stop when we did not find more docker images to rebuild and + # therefore cannot find more extra events. + while prev_builds_count != len(builds): + prev_builds_count = len(builds) + extra_events = self._find_events_to_include(db_event, builds) + log.info("Extra events: %r", extra_events) + for ev in extra_events: + if ev in seen_extra_events: + continue + seen_extra_events.append(ev) + builds = self._record_images_to_rebuild(ev, event, builds) + repo_urls.append(self._prepare_yum_repo(ev)) + + # Remove duplicates from repo_urls. + repo_urls = list(set(repo_urls)) + + # Log what we are going to rebuild + self._log_images_to_rebuild(builds) + log.info("Following repositories will be used for the rebuild:") + for url in repo_urls: + log.info(" - %s", url) + + # TODO: Rebuild first batch. + + return [] + + def _prepare_yum_repo(self, db_event): + """ + Prepare a yum repo for rebuild + + Run a compose in ODCS to contain required RPMs for rebuilding images + later. + """ + + errata_id = int(db_event.search_key) + + packages = [] + errata = Errata(conf.errata_tool_server_url) + builds = errata.get_builds(errata_id) + compose_source = None + for nvr in builds: + packages += self._get_packages_for_compose(nvr) + source = self._get_compose_source(nvr) + if compose_source and compose_source != source: + # TODO: Handle this by generating two ODCS composes + log.error("Packages for errata advisory %d found in multiple " + "different tags", errata_id) + return + else: + compose_source = source + + odcs = ODCS(conf.odcs_server_url, auth_mech=AuthMech.Kerberos, + verify_ssl=conf.odcs_verify_ssl) + + if compose_source is None: + log.error('Builds for errata %d are not the latest build in its ' + 'all tags.', errata_id) + return + + log.info('Generate new compose for rebuild: ' + 'source: %s, source type: %s, packages: %s', + compose_source, 'tag', packages) + + new_compose = odcs.new_compose(compose_source, + 'tag', + packages=packages) + compose_id = new_compose['id'] + + log.info('Waiting for ODCS to finish the compose: %d', compose_id) + + while True: + time.sleep(1) + + new_compose = odcs.get_compose(compose_id) + state = new_compose['state'] + if state == 0: # waiting for generating compose + log.info('Waiting for generating new compose') + elif state == 1: # generating in progress + log.info('ODCS is generating the compose') + elif state == 4: # Failed to generate compose + log.error('ODCS fails to generate compose: %d', compose_id) + log.error('Please consult ODCS to see what is wrong with it') + return + elif state == 2: # Succeed to generate compose + log.info('ODCS has finished to generate compose. Continue to rebuild') + break + else: + log.error('Got unexpected compose state {0} from ODCS.'.format(state)) + return + + log.info('Repo URL containing packages used to rebuild container: %s', + new_compose['result_repo']) + + return new_compose['result_repo'] + + def _get_packages_for_compose(self, nvr): + """Get RPMs of current build NVR""" + with koji_service(conf.koji_profile, log) as session: + rpms = session.get_build_rpms(nvr) + return list(set([rpm['name'] for rpm in rpms])) + + def _get_compose_source(self, nvr): + """Get tag from which to collect packages to compose""" + with koji_service(conf.koji_profile, log) as service: + tag = [tag['name'] for tag in service.session.listTags(nvr) + if tag['name'].endswith('-candidate')][0] + latest_build = service.session.listTagged( + tag, + latest=True, + package=koji.parse_NVR(nvr)['name']) + if latest_build and latest_build[0]['nvr'] == nvr: + return tag + + def _log_images_to_rebuild(self, builds): + """ + Logs the information about images to rebuilt using log.info(...). + :param builds dict: list of docker images to build as returned by + _record_images_to_rebuild(...). + """ + log.info('Found docker images to rebuild in following order:') + batch = 0 + printed = [] + while len(printed) != len(builds.values()): + log.info(' Batch %d:', batch) + for build in builds.values(): + if (((build.dep_on and build.dep_on.name in printed) + or (not build.dep_on and batch == 0)) + and not build.name in printed): + args = json.loads(build.build_args) + based_on = "based on %s" % args["parent"] \ + if args["parent"] else "base image" + log.info(' - %s#%s (%s)' % + (args["repository"], args["commit"], based_on)) + printed.append(build.name) + + batch += 1 + + def _find_events_to_include(self, db_event, builds): + """ + Find out all unreleased events which built some image which is also + planned to be build as part of current image rebuild. + + :param db_event Event: Database representation of + ErrataAdvisoryRPMsSignedEvent. + :param builds dict: list of docker images to build as returned by + _record_images_to_rebuild(...). + """ + events_to_include = [] + for ev in Event.get_unreleased(db.session): + for build in ev.builds: + # Skip non IMAGE builds + if (build.type != ArtifactType.IMAGE.value + or ev.message_id == db_event.message_id): + continue + + if build.name in builds: + events_to_include.append(ev) + break + + return events_to_include + + def _record_batches(self, batches, event, builds=None): + """ + Records the images from batches to database. + :param batches list: Output of _find_images_to_rebuild(...). + :param event ErrataAdvisoryRPMsSignedEvent: The main event this handler + is currently handling. + :param builds dict: list of docker images to build as returned by + _record_images_to_rebuild(...). + """ + + # Used as tmp dict with {brew_buil_id: ArtifactBuild, ...} mapping. + builds = builds or {} + + for batch in batches: + for image in batch: + name = image["brew"]["build"] + if name in builds: + log.debug("Skipping recording build %s, " + "it is already in db", name) + continue + log.debug("Recording %s", name) + parent_name = image["parent"]["brew"]["build"] \ + if image["parent"] else None + dep_on = builds[parent_name] if parent_name in builds else None + build = self.record_build( + event, name, ArtifactType.IMAGE, 0, dep_on, + ArtifactBuildState.PLANNED.value) + + build_args = {} + build_args["repository"] = image["repository"] + build_args["commit"] = image["commit"] + build_args["parent"] = parent_name + build.build_args = json.dumps(build_args) + db.session.commit() + + builds[name] = build + + return builds + + def _record_images_to_rebuild(self, db_event, event, builds=None): + """ + Finds and records to DB the list of Docker images to rebuild based + on the particular ErrataAdvisoryRPMsSignedEvent. + + :param db_event Event: Database representation of + ErrataAdvisoryRPMsSignedEvent. + :param event ErrataAdvisoryRPMsSignedEvent: The main event this handler + is currently handling. + :param builds dict: list of docker images to build as returned by + previous calls of _record_images_to_rebuild(...). + """ + + errata = Errata(conf.errata_tool_server_url) + errata_id = int(db_event.search_key) + + # Use the errata_id to find out Pulp repository IDs from Errata Tool + # and furthermore get content_sets from Pulp where signed RPM will end + # up eventually when advisories are shipped. + pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id))) + + pulp = Pulp(server_url=conf.pulp_server_url, + username=conf.pulp_username, + password=conf.pulp_password) + content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids) + + log.info('RPM will end up within content sets %s', content_sets) + + # Query images from LightBlue by signed RPM's srpm name and found + # content sets + lb = LightBlue(server_url=conf.lightblue_server_url, + cert=conf.lightblue_certificate, + private_key=conf.lightblue_private_key) + + # For each RPM build in Errata advisory, find the list of Docker + # images containing this RPM and record it to DB. + builds = builds or {} + nvrs = errata.get_builds(errata_id) + for nvr in nvrs: + srpm_name = self._find_build_srpm_name(nvr) + batches = lb.find_images_to_rebuild(srpm_name, content_sets) + builds = self._record_batches(batches, event, builds) + return builds + + def _find_build_srpm_name(self, build_nvr): + """Find srpm name from a build""" + with koji_service(conf.koji_profile, log) as session: + rpm_infos = session.get_build_rpms(build_nvr, arches='src') + if not rpm_infos: + raise ValueError( + 'Build {} does not have a SRPM, although this should not ' + 'happen in practice.'.format(build_nvr)) + return rpm_infos[0]['name'] diff --git a/freshmaker/handlers/errata/errata_advisory_state_changed.py b/freshmaker/handlers/errata/errata_advisory_state_changed.py new file mode 100644 index 0000000..64f0bcf --- /dev/null +++ b/freshmaker/handlers/errata/errata_advisory_state_changed.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker import log +from freshmaker import db +from freshmaker.events import ( + ErrataAdvisoryStateChangedEvent, ErrataAdvisoryRPMsSignedEvent) +from freshmaker.models import Event, EVENT_TYPES +from freshmaker.handlers import BaseHandler + + +class ErrataAdvisoryStateChangedHandler(BaseHandler): + """Rebuild container when a dependecy container is built in Brew""" + + name = 'ErrataAdvisoryStateChangedHandler' + + def can_handle(self, event): + return isinstance(event, ErrataAdvisoryStateChangedEvent) + + def handle(self, event): + """ + When build container task state changed in brew, update build state in db and + rebuild containers depend on the success build as necessary. + """ + + errata_id = event.errata_id + state = event.state + if state != "SHIPPED_LIVE": + log.debug("Ignoring Errata advisory %d state change to %s, " + "because it is not SHIPPED_LIVE", errata_id, state) + return [] + + # check db to see whether this advisory exists in db + db_event = db.session.query(Event).filter_by( + event_type_id=EVENT_TYPES[ErrataAdvisoryRPMsSignedEvent], + search_key=str(errata_id)).one_or_none() + if not db_event: + log.debug("Ignoring Errata advisory %d - it does not exist in " + "Freshmaker db.", errata_id) + return [] + + db_event.released = True + db.session.commit() + log.info("Errata advisory %d is now marked as released", errata_id) diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index fcee9fe..9c61fb6 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -22,6 +22,7 @@ # Written by Chenxiong Qi import contextlib +from freshmaker import log class KojiService(object): @@ -99,6 +100,7 @@ class KojiService(object): return task_id def get_build_rpms(self, build_nvr, arches=None): + log.info("get_build_rpms %r", build_nvr) build_info = self.session.getBuild(build_nvr) return self.session.listRPMs(buildID=build_info['id'], arches=arches) diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 34c953a..f8dcc18 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -130,7 +130,7 @@ class ContainerImage(dict): srpm_nevra = None if "parsed_data" in self and "rpm_manifest" in self["parsed_data"]: for rpm in self["parsed_data"]["rpm_manifest"]: - if rpm["srpm_name"] == srpm_name: + if "srpm_name" in rpm and rpm["srpm_name"] == srpm_name: srpm_nevra = rpm['srpm_nevra'] break diff --git a/freshmaker/logger.py b/freshmaker/logger.py index b667049..b144519 100644 --- a/freshmaker/logger.py +++ b/freshmaker/logger.py @@ -71,7 +71,7 @@ def init_logging(conf): """ Initializes logging according to configuration file. """ - log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' + log_format = '%(levelname)s - %(message)s' log_backend = conf.log_backend if not log_backend or len(log_backend) == 0 or log_backend == "console": diff --git a/freshmaker/migrations/versions/43b3c6580af7_.py b/freshmaker/migrations/versions/43b3c6580af7_.py new file mode 100644 index 0000000..88c62a9 --- /dev/null +++ b/freshmaker/migrations/versions/43b3c6580af7_.py @@ -0,0 +1,26 @@ +"""empty message + +Revision ID: 43b3c6580af7 +Revises: 8d2e9cd99c54 +Create Date: 2017-08-15 10:29:33.224878 + +""" + +# revision identifiers, used by Alembic. +revision = '43b3c6580af7' +down_revision = '8d2e9cd99c54' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('events', sa.Column('released', sa.Boolean(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('events', 'released') + # ### end Alembic commands ### diff --git a/freshmaker/models.py b/freshmaker/models.py index 3225105..735d1a0 100644 --- a/freshmaker/models.py +++ b/freshmaker/models.py @@ -32,7 +32,8 @@ from freshmaker.types import ArtifactType, ArtifactBuildState from freshmaker.events import ( MBSModuleStateChangeEvent, GitModuleMetadataChangeEvent, GitRPMSpecChangeEvent, TestingEvent, GitDockerfileChangeEvent, - BodhiUpdateCompleteStableEvent, KojiTaskStateChangeEvent, BrewSignRPMEvent) + BodhiUpdateCompleteStableEvent, KojiTaskStateChangeEvent, BrewSignRPMEvent, + ErrataAdvisoryRPMsSignedEvent) EVENT_TYPES = { MBSModuleStateChangeEvent: 0, @@ -43,6 +44,7 @@ EVENT_TYPES = { BodhiUpdateCompleteStableEvent: 5, KojiTaskStateChangeEvent: 6, BrewSignRPMEvent: 7, + ErrataAdvisoryRPMsSignedEvent: 8, } INVERSE_EVENT_TYPES = {v: k for k, v in EVENT_TYPES.items()} @@ -63,28 +65,36 @@ class Event(FreshmakerBase): # Event type id defined in EVENT_TYPES - ID of class inherited from # BaseEvent class - used when searching for events of particular type. event_type_id = db.Column(db.Integer, nullable=False) + # True when the Event is already released and we do not have to include + # it in the future rebuilds of artifacts. + released = db.Column(db.Boolean, default=True) # List of builds associated with this Event. builds = relationship("ArtifactBuild", back_populates="event") @classmethod - def create(cls, session, message_id, search_key, event_type): + def create(cls, session, message_id, search_key, event_type, released=True): if event_type in EVENT_TYPES: event_type = EVENT_TYPES[event_type] event = cls( message_id=message_id, search_key=search_key, - event_type_id=event_type + event_type_id=event_type, + released=released, ) session.add(event) return event @classmethod - def get_or_create(cls, session, message_id, search_key, event_type): + def get_or_create(cls, session, message_id, search_key, event_type, released=True): instance = session.query(cls).filter_by(message_id=message_id).first() if instance: return instance - return cls.create(session, message_id, search_key, event_type) + return cls.create(session, message_id, search_key, event_type, released) + + @classmethod + def get_unreleased(cls, session): + return session.query(cls).filter_by(released=False).all() @property def event_type(self): @@ -99,7 +109,7 @@ class Event(FreshmakerBase): "message_id": self.message_id, "search_key": self.search_key, "event_type_id": self.event_type_id, - "builds": [b.id for b in self.builds], + "builds": [b.json() for b in self.builds], } @@ -168,7 +178,10 @@ class ArtifactBuild(FreshmakerBase): "id": self.id, "name": self.name, "type": self.type, + "type_name": ArtifactType(self.type).name, "state": self.state, + "state_name": ArtifactBuildState(self.state).name, + "dep_on": self.dep_on.name if self.dep_on else None, "time_submitted": self.time_submitted, "time_completed": self.time_completed, "event_id": self.event_id, diff --git a/freshmaker/parsers/errata/__init__.py b/freshmaker/parsers/errata/__init__.py new file mode 100644 index 0000000..9e316ea --- /dev/null +++ b/freshmaker/parsers/errata/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from .state_change import ErrataAdvisoryStateChangedParser # noqa diff --git a/freshmaker/parsers/errata/state_change.py b/freshmaker/parsers/errata/state_change.py new file mode 100644 index 0000000..df2e7f2 --- /dev/null +++ b/freshmaker/parsers/errata/state_change.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from freshmaker.parsers import BaseParser +from freshmaker.events import ErrataAdvisoryStateChangedEvent + + +class ErrataAdvisoryStateChangedParser(BaseParser): + """ + Parser parsing errata.activity.status. + """ + + name = "ErrataAdvisoryStateChangedParser" + topic_suffixes = ["eng.errata.activity.status"] + + def can_parse(self, topic, msg): + return any([topic.endswith(s) for s in self.topic_suffixes]) + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + inner_msg = msg.get('msg') + status = inner_msg.get('errata_status') + errata_id = int(inner_msg.get('errata_id')) + + return ErrataAdvisoryStateChangedEvent(msg_id, errata_id, status) From ad8eac9e0e39a70cffe8f4e64636b43edc5dafbc Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 18 2017 09:20:59 +0000 Subject: [PATCH 2/2] Add tests for restructured Docker image handling. --- diff --git a/freshmaker/events.py b/freshmaker/events.py index 61741d5..6b74b60 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -248,9 +248,10 @@ class KojiTaskStateChangeEvent(BaseEvent): self.task_id = task_id self.task_state = task_state + class ErrataAdvisoryStateChangedEvent(BaseEvent): """ - Represents change od Errata Advisory status. + Represents change of Errata Advisory status. """ def __init__(self, msg_id, errata_id, state): @@ -258,6 +259,7 @@ class ErrataAdvisoryStateChangedEvent(BaseEvent): self.errata_id = errata_id self.state = state + class ErrataAdvisoryRPMsSignedEvent(BaseEvent): """ Event when all RPMs in Errata advisory are signed. @@ -272,6 +274,7 @@ class ErrataAdvisoryRPMsSignedEvent(BaseEvent): def search_key(self): return str(self.errata_id) + class BrewSignRPMEvent(BaseEvent): """ Represents the message sent by Brew when RPM is signed. diff --git a/freshmaker/handlers/brew/sign_rpm.py b/freshmaker/handlers/brew/sign_rpm.py index ce6fb89..66f1fc6 100644 --- a/freshmaker/handlers/brew/sign_rpm.py +++ b/freshmaker/handlers/brew/sign_rpm.py @@ -21,26 +21,12 @@ # # Written by Chenxiong Qi -import json -import koji -import time - -from itertools import chain - from freshmaker import conf from freshmaker import log -from freshmaker import db from freshmaker.events import BrewSignRPMEvent, ErrataAdvisoryRPMsSignedEvent from freshmaker.handlers import BaseHandler -from freshmaker.kojiservice import koji_service -from freshmaker.lightblue import LightBlue -from freshmaker.pulp import Pulp from freshmaker.errata import Errata -from freshmaker.types import ArtifactType, ArtifactBuildState -from freshmaker.models import Event - -from odcs.client.odcs import ODCS -from odcs.client.odcs import AuthMech +from freshmaker.types import ArtifactType class BrewSignRPMHandler(BaseHandler): diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index 4242152..a461282 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -26,8 +26,6 @@ import json import koji import time -from itertools import chain - from freshmaker import conf from freshmaker import log from freshmaker import db @@ -79,7 +77,8 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): # ErrataAdvisoryRPMsSignedEvent event. builds = self._record_images_to_rebuild(db_event, event) if not builds: - log.info('Not found docker images to rebuild.') + log.info('No container images to rebuild for advisory %r', + event.errata_name) return [] # Generate the ODCS compose with RPMs from the current advisory. @@ -224,14 +223,21 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): while len(printed) != len(builds.values()): log.info(' Batch %d:', batch) for build in builds.values(): - if (((build.dep_on and build.dep_on.name in printed) - or (not build.dep_on and batch == 0)) - and not build.name in printed): + # Print build only if: + # a) It depends on other build, but this dependency has not + # been printed yet or ... + # b) ... it does not depend on other build and we are printing + # batch 0 - this handles the base images + # In call cases, print only builds which have not been printed + # so far. + if (build.name not in printed and + ((build.dep_on and build.dep_on.name in printed) or + (not build.dep_on and batch == 0))): args = json.loads(build.build_args) based_on = "based on %s" % args["parent"] \ if args["parent"] else "base image" log.info(' - %s#%s (%s)' % - (args["repository"], args["commit"], based_on)) + (args["repository"], args["commit"], based_on)) printed.append(build.name) batch += 1 @@ -250,8 +256,8 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): for ev in Event.get_unreleased(db.session): for build in ev.builds: # Skip non IMAGE builds - if (build.type != ArtifactType.IMAGE.value - or ev.message_id == db_event.message_id): + if (build.type != ArtifactType.IMAGE.value or + ev.message_id == db_event.message_id): continue if build.name in builds: diff --git a/freshmaker/models.py b/freshmaker/models.py index 735d1a0..e89bb1c 100644 --- a/freshmaker/models.py +++ b/freshmaker/models.py @@ -67,6 +67,8 @@ class Event(FreshmakerBase): event_type_id = db.Column(db.Integer, nullable=False) # True when the Event is already released and we do not have to include # it in the future rebuilds of artifacts. + # This is currently only used for internal Docker images rebuilds, but in + # the future might be used even for modules or Fedora Docker images. released = db.Column(db.Boolean, default=True) # List of builds associated with this Event. diff --git a/tests/test_brew_sign_rpm_handler.py b/tests/test_brew_sign_rpm_handler.py index be441dc..9815afa 100644 --- a/tests/test_brew_sign_rpm_handler.py +++ b/tests/test_brew_sign_rpm_handler.py @@ -21,73 +21,38 @@ # # Written by Chenxiong Qi -import six -import pytest import unittest -import json -from mock import patch, MagicMock, PropertyMock, Mock +from mock import patch, MagicMock, PropertyMock from freshmaker.handlers.brew.sign_rpm import BrewSignRPMHandler from freshmaker.errata import ErrataAdvisory -from freshmaker import db, events -from freshmaker.models import Event -from freshmaker.types import ArtifactBuildState, ArtifactType +class TestBrewSignHandler(unittest.TestCase): + """Test BrewSignRPMHandler.handle""" -@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') -class TestFindBuildSrpmName(unittest.TestCase): - """Test BrewSignRPMHandler._find_build_srpm_name""" - - @patch('koji.ClientSession') - def test_find_srpm_name(self, ClientSession): - session = ClientSession.return_value - session.getBuild.return_value = { - 'build_id': 439408, - 'id': 439408, - 'name': 'bind-dyndb-ldap', - 'nvr': 'bind-dyndb-ldap-2.3-8.el6', - } - session.listRPMs.return_value = [{ - 'arch': 'src', - 'name': 'bind-dyndb-ldap', - 'nvr': 'bind-dyndb-ldap-2.3-8.el6', - }] - - handler = BrewSignRPMHandler() - srpm_name = handler._find_build_srpm_name('bind-dyndb-ldap-2.3-8.el6') - - session.getBuild.assert_called_once_with('bind-dyndb-ldap-2.3-8.el6') - session.listRPMs.assert_called_once_with(buildID=439408, arches='src') - self.assertEqual('bind-dyndb-ldap', srpm_name) - - @patch('koji.ClientSession') - def test_error_if_no_srpm_in_build(self, ClientSession): - session = ClientSession.return_value - session.getBuild.return_value = { - 'build_id': 439408, - 'id': 439408, - 'name': 'bind-dyndb-ldap', - 'nvr': 'bind-dyndb-ldap-2.3-8.el6', - } - session.listRPMs.return_value = [] + @patch('freshmaker.errata.Errata.advisories_from_event') + @patch('freshmaker.errata.Errata.builds_signed') + @patch("freshmaker.config.Config.handler_build_whitelist", + new_callable=PropertyMock, return_value={ + "BrewSignRPMHandler": {"image": [{"advisory_name": "RHSA-.*"}]}}) + def test_return_value(self, handler_build_whitelist, builds_signed, + advisories_from_event): + """ + Tests that handle method returns ErrataAdvisoryRPMsSignedEvent. + """ + advisories_from_event.return_value = [ + ErrataAdvisory(123, "RHSA-2017", "REL_PREP")] + builds_signed.return_value = True + event = MagicMock() handler = BrewSignRPMHandler() + ret = handler.handle(event) - self.assertRaisesRegexp( - ValueError, - 'Build bind-dyndb-ldap-2.3-8.el6 does not have a SRPM', - handler._find_build_srpm_name, - 'bind-dyndb-ldap-2.3-8.el6', - ) - - session.getBuild.assert_called_once_with('bind-dyndb-ldap-2.3-8.el6') - session.listRPMs.assert_called_once_with(buildID=439408, arches='src') - - -class TestAllowBuild(unittest.TestCase): - """Test BrewSignRPMHandler.allow_build""" + self.assertTrue(len(ret), 1) + self.assertEqual(ret[0].errata_name, "RHSA-2017") + self.assertEqual(ret[0].errata_id, 123) @patch('freshmaker.errata.Errata.advisories_from_event') @patch('freshmaker.errata.Errata.builds_signed') @@ -105,8 +70,9 @@ class TestAllowBuild(unittest.TestCase): event = MagicMock() handler = BrewSignRPMHandler() - handler.handle(event) + ret = handler.handle(event) + self.assertTrue(not ret) builds_signed.assert_not_called() @patch('freshmaker.errata.Errata.advisories_from_event') @@ -191,226 +157,3 @@ class TestAllowBuild(unittest.TestCase): handler.handle(event) builds_signed.assert_not_called() - - -class TestBatches(unittest.TestCase): - """Test handling of batches""" - - def setUp(self): - db.session.remove() - db.drop_all() - db.create_all() - db.session.commit() - - def tearDown(self): - db.session.remove() - db.drop_all() - db.session.commit() - - def _mock_build(self, build, parent=None): - if parent: - parent = {"brew": {"build": parent}} - return {'brew': {'build': build}, 'repository': build + '_repo', - 'commit': build + '_123', 'parent': parent} - - def test_batches_records(self): - """ - Tests that batches are properly recorded in DB. - """ - # Creates following tree: - # shared_parent - # |- child1_parent3 - # |- child1_parent2 - # |- child1_parent1 - # |- child1 - # |- child2_parent2 - # |- child2_parent1 - # |- child2 - batches = [[self._mock_build("shared_parent")], - [self._mock_build("child1_parent3", "shared_parent"), - self._mock_build("child2_parent2", "shared_parent")], - [self._mock_build("child1_parent2", "child1_parent3"), - self._mock_build("child2_parent1", "child2_parent2")], - [self._mock_build("child1_parent1", "child1_parent2"), - self._mock_build("child2", "child2_parent1")], - [self._mock_build("child1", "child1_parent1")]] - - # Flat list of images from batches with brew build id as a key. - images = {} - for batch in batches: - for image in batch: - images[image['brew']['build']] = image - - # Record the batches. - event = events.BrewSignRPMEvent("123", "openssl-1.1.0-1") - handler = BrewSignRPMHandler() - handler._record_batches(batches, event) - - # Check that the images have proper data in proper db columns. - e = db.session.query(Event).filter(Event.id == 1).one() - for build in e.builds: - self.assertEqual(build.state, ArtifactBuildState.PLANNED.value) - self.assertEqual(build.type, ArtifactType.IMAGE.value) - - image = images[build.name] - if image['parent']: - self.assertEqual(build.dep_on.name, image['parent']['brew']['build']) - else: - self.assertEqual(build.dep_on, None) - - args = json.loads(build.build_args) - self.assertEqual(args["repository"], build.name + "_repo") - self.assertEqual(args["commit"], build.name + "_123") - self.assertEqual(args["parent"], - build.dep_on.name if build.dep_on else None) - - -class TestGetPackagesForCompose(unittest.TestCase): - """Test BrewSignRPMHandler._get_packages_for_compose""" - - @patch('freshmaker.kojiservice.KojiService.get_build_rpms') - def test_get_packages(self, get_build_rpms): - get_build_rpms.return_value = [ - { - 'id': 4672404, - 'arch': 'src', - 'name': 'chkconfig', - 'release': '1.el7_3.1', - 'version': '1.7.2', - 'nvr': 'chkconfig-1.7.2-1.el7_3.1', - }, - { - 'id': 4672405, - 'arch': 'ppc', - 'name': 'chkconfig', - 'release': '1.el7_3.1', - 'version': '1.7.2', - 'nvr': 'chkconfig-1.7.2-1.el7_3.1', - }, - { - 'id': 4672420, - 'arch': 'i686', - 'name': 'chkconfig-debuginfo', - 'release': '1.el7_3.1', - 'version': '1.7.2', - 'nvr': 'chkconfig-debuginfo-1.7.2-1.el7_3.1', - } - ] - - build_nvr = 'chkconfig-1.7.2-1.el7_3.1' - handler = BrewSignRPMHandler() - packages = handler._get_packages_for_compose(build_nvr) - - get_build_rpms.assert_called_once_with(build_nvr) - - self.assertEqual(set(['chkconfig', 'chkconfig-debuginfo']), - set(packages)) - - -class TestGetComposeSource(unittest.TestCase): - """Test BrewSignRPMHandler._get_compose_source""" - - @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) - def test_get_tag(self, session): - session.listTags.return_value = [ - { - 'id': 10974, - 'name': 'rhscl-3.0-rhel-6-candidate', - }, - { - 'id': 11030, - 'name': 'rhscl-3.0-rhel-6-pending', - }, - { - 'id': 11425, - 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', - } - ] - session.listTagged.return_value = [ - { - 'build_id': 568228, - 'nvr': 'rh-postgresql96-3.0-9.el6', - } - ] - - handler = BrewSignRPMHandler() - tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') - self.assertEqual('rhscl-3.0-rhel-6-candidate', tag) - - @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) - def test_get_None_if_tag_has_new_build(self, session): - session.listTags.return_value = [ - { - 'id': 10974, - 'name': 'rhscl-3.0-rhel-6-candidate', - }, - { - 'id': 11030, - 'name': 'rhscl-3.0-rhel-6-pending', - }, - { - 'id': 11425, - 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', - } - ] - session.listTagged.return_value = [ - { - 'build_id': 568228, - 'nvr': 'rh-postgresql96-3.0-10.el6', - } - ] - - handler = BrewSignRPMHandler() - tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') - self.assertEqual(None, tag) - - -class TestPrepareYumRepo(unittest.TestCase): - """Test BrewSignRPMHandler._prepare_yum_repo""" - - @patch('freshmaker.handlers.brew.sign_rpm.ODCS') - @patch('freshmaker.handlers.brew.sign_rpm.' - 'BrewSignRPMHandler._get_packages_for_compose') - @patch('freshmaker.handlers.brew.sign_rpm.' - 'BrewSignRPMHandler._get_compose_source') - @patch('time.sleep') - def test_get_repo_url_when_succeed_to_generate_compose( - self, sleep, _get_compose_source, _get_packages_for_compose, ODCS): - _get_packages_for_compose.return_value = ['httpd', 'httpd-debuginfo'] - _get_compose_source.return_value = 'rhel-7.2-candidate' - ODCS.return_value.new_compose.return_value = { - "id": 3, - "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", - "source": "f26", - "source_type": 1, - "state": 0, - "state_name": "wait", - } - ODCS.return_value.get_compose.return_value = { - "id": 3, - "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", - "source": "f26", - "source_type": 1, - "state": 2, - "state_name": "done", - } - - event = Mock(nvr='httpd-0.1-1.f26') - handler = BrewSignRPMHandler() - repo_url = handler._prepare_yum_repo(event) - - _get_compose_source.assert_called_once_with(event.nvr) - _get_packages_for_compose.assert_called_once_with(event.nvr) - - # Ensure new_compose is called to request a new compose - ODCS.return_value.new_compose.assert_called_once_with( - 'rhel-7.2-candidate', 'tag', packages=['httpd', 'httpd-debuginfo']) - - # Ensure get_compose is called once in order to get lates state and see - # if it still needs to wait for ODCS - ODCS.return_value.get_compose.assert_called_once_with(3) - - # We should get the right repo URL eventually - self.assertEqual( - 'http://localhost/composes/latest-odcs-3-1/compose/Temporary', - repo_url) diff --git a/tests/test_errata_advisory_state_changed.py b/tests/test_errata_advisory_state_changed.py new file mode 100644 index 0000000..0282961 --- /dev/null +++ b/tests/test_errata_advisory_state_changed.py @@ -0,0 +1,458 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +import six +import pytest +import unittest +import json + +from mock import patch, MagicMock, PropertyMock, Mock + +from freshmaker.handlers.errata import ErrataAdvisoryRPMsSignedHandler +from freshmaker.events import ErrataAdvisoryRPMsSignedEvent + +from freshmaker import db, events +from freshmaker.models import Event, ArtifactBuild +from freshmaker.types import ArtifactBuildState, ArtifactType + + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class TestFindBuildSrpmName(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler._find_build_srpm_name""" + + @patch('koji.ClientSession') + def test_find_srpm_name(self, ClientSession): + session = ClientSession.return_value + session.getBuild.return_value = { + 'build_id': 439408, + 'id': 439408, + 'name': 'bind-dyndb-ldap', + 'nvr': 'bind-dyndb-ldap-2.3-8.el6', + } + session.listRPMs.return_value = [{ + 'arch': 'src', + 'name': 'bind-dyndb-ldap', + 'nvr': 'bind-dyndb-ldap-2.3-8.el6', + }] + + handler = ErrataAdvisoryRPMsSignedHandler() + srpm_name = handler._find_build_srpm_name('bind-dyndb-ldap-2.3-8.el6') + + session.getBuild.assert_called_once_with('bind-dyndb-ldap-2.3-8.el6') + session.listRPMs.assert_called_once_with(buildID=439408, arches='src') + self.assertEqual('bind-dyndb-ldap', srpm_name) + + @patch('koji.ClientSession') + def test_error_if_no_srpm_in_build(self, ClientSession): + session = ClientSession.return_value + session.getBuild.return_value = { + 'build_id': 439408, + 'id': 439408, + 'name': 'bind-dyndb-ldap', + 'nvr': 'bind-dyndb-ldap-2.3-8.el6', + } + session.listRPMs.return_value = [] + + handler = ErrataAdvisoryRPMsSignedHandler() + + self.assertRaisesRegexp( + ValueError, + 'Build bind-dyndb-ldap-2.3-8.el6 does not have a SRPM', + handler._find_build_srpm_name, + 'bind-dyndb-ldap-2.3-8.el6', + ) + + session.getBuild.assert_called_once_with('bind-dyndb-ldap-2.3-8.el6') + session.listRPMs.assert_called_once_with(buildID=439408, arches='src') + + +class TestAllowBuild(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler.allow_build""" + + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + @patch("freshmaker.handlers.errata.ErrataAdvisoryRPMsSignedHandler." + "_record_images_to_rebuild", return_value=[]) + @patch("freshmaker.config.Config.handler_build_whitelist", + new_callable=PropertyMock, return_value={ + "ErrataAdvisoryRPMsSignedHandler": {"image": [{"advisory_name": "RHSA-.*"}]}}) + def test_allow_build_false(self, handler_build_whitelist, record_images): + """ + Tests that allow_build filters out advisories based on advisory_name. + """ + event = ErrataAdvisoryRPMsSignedEvent("123", "RHBA-2017", 123, "") + handler = ErrataAdvisoryRPMsSignedHandler() + handler.handle(event) + + record_images.assert_not_called() + + @patch("freshmaker.handlers.errata.ErrataAdvisoryRPMsSignedHandler." + "_record_images_to_rebuild", return_value=[]) + @patch("freshmaker.config.Config.handler_build_whitelist", + new_callable=PropertyMock, return_value={ + "ErrataAdvisoryRPMsSignedHandler": {"image": [{"advisory_name": "RHSA-.*"}]}}) + def test_allow_build_true(self, handler_build_whitelist, record_images): + """ + Tests that allow_build does not filter out advisories based on + advisory_name. + """ + event = ErrataAdvisoryRPMsSignedEvent("123", "RHSA-2017", 123, "") + handler = ErrataAdvisoryRPMsSignedHandler() + handler.handle(event) + + record_images.assert_called_once() + + @patch("freshmaker.handlers.errata.ErrataAdvisoryRPMsSignedHandler." + "_record_images_to_rebuild", return_value=[]) + @patch( + "freshmaker.config.Config.handler_build_whitelist", + new_callable=PropertyMock, + return_value={ + "ErrataAdvisoryRPMsSignedHandler": { + "image": [{ + "advisory_security_impact": [ + "Normal", "Important" + ] + }] + } + }) + def test_allow_security_impact_important_true( + self, handler_build_whitelist, record_images): + """ + Tests that allow_build does not filter out advisories based on + advisory_security_impact. + """ + event = ErrataAdvisoryRPMsSignedEvent("123", "RHSA-2017", 123, + "Important") + handler = ErrataAdvisoryRPMsSignedHandler() + handler.handle(event) + + record_images.assert_called_once() + + @patch("freshmaker.handlers.errata.ErrataAdvisoryRPMsSignedHandler." + "_record_images_to_rebuild", return_value=[]) + @patch( + "freshmaker.config.Config.handler_build_whitelist", + new_callable=PropertyMock, + return_value={ + "ErrataAdvisoryRPMsSignedHandler": { + "image": [{ + "advisory_security_impact": [ + "Normal", "Important" + ] + }] + } + }) + def test_allow_security_impact_important_false( + self, handler_build_whitelist, record_images): + """ + Tests that allow_build dost filter out advisories based on + advisory_security_impact. + """ + event = ErrataAdvisoryRPMsSignedEvent("123", "RHSA-2017", 123, "None") + handler = ErrataAdvisoryRPMsSignedHandler() + handler.handle(event) + + record_images.assert_not_called() + + +class TestBatches(unittest.TestCase): + """Test handling of batches""" + + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def _mock_build(self, build, parent=None): + if parent: + parent = {"brew": {"build": parent}} + return {'brew': {'build': build}, 'repository': build + '_repo', + 'commit': build + '_123', 'parent': parent} + + def test_batches_records(self): + """ + Tests that batches are properly recorded in DB. + """ + # Creates following tree: + # shared_parent + # |- child1_parent3 + # |- child1_parent2 + # |- child1_parent1 + # |- child1 + # |- child2_parent2 + # |- child2_parent1 + # |- child2 + batches = [[self._mock_build("shared_parent")], + [self._mock_build("child1_parent3", "shared_parent"), + self._mock_build("child2_parent2", "shared_parent")], + [self._mock_build("child1_parent2", "child1_parent3"), + self._mock_build("child2_parent1", "child2_parent2")], + [self._mock_build("child1_parent1", "child1_parent2"), + self._mock_build("child2", "child2_parent1")], + [self._mock_build("child1", "child1_parent1")]] + + # Flat list of images from batches with brew build id as a key. + images = {} + for batch in batches: + for image in batch: + images[image['brew']['build']] = image + + # Record the batches. + event = events.BrewSignRPMEvent("123", "openssl-1.1.0-1") + handler = ErrataAdvisoryRPMsSignedHandler() + handler._record_batches(batches, event) + + # Check that the images have proper data in proper db columns. + e = db.session.query(Event).filter(Event.id == 1).one() + for build in e.builds: + self.assertEqual(build.state, ArtifactBuildState.PLANNED.value) + self.assertEqual(build.type, ArtifactType.IMAGE.value) + + image = images[build.name] + if image['parent']: + self.assertEqual(build.dep_on.name, image['parent']['brew']['build']) + else: + self.assertEqual(build.dep_on, None) + + args = json.loads(build.build_args) + self.assertEqual(args["repository"], build.name + "_repo") + self.assertEqual(args["commit"], build.name + "_123") + self.assertEqual(args["parent"], + build.dep_on.name if build.dep_on else None) + + +class TestGetPackagesForCompose(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler._get_packages_for_compose""" + + @patch('freshmaker.kojiservice.KojiService.get_build_rpms') + def test_get_packages(self, get_build_rpms): + get_build_rpms.return_value = [ + { + 'id': 4672404, + 'arch': 'src', + 'name': 'chkconfig', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-1.7.2-1.el7_3.1', + }, + { + 'id': 4672405, + 'arch': 'ppc', + 'name': 'chkconfig', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-1.7.2-1.el7_3.1', + }, + { + 'id': 4672420, + 'arch': 'i686', + 'name': 'chkconfig-debuginfo', + 'release': '1.el7_3.1', + 'version': '1.7.2', + 'nvr': 'chkconfig-debuginfo-1.7.2-1.el7_3.1', + } + ] + + build_nvr = 'chkconfig-1.7.2-1.el7_3.1' + handler = ErrataAdvisoryRPMsSignedHandler() + packages = handler._get_packages_for_compose(build_nvr) + + get_build_rpms.assert_called_once_with(build_nvr) + + self.assertEqual(set(['chkconfig', 'chkconfig-debuginfo']), + set(packages)) + + +class TestGetComposeSource(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler._get_compose_source""" + + @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) + def test_get_tag(self, session): + session.listTags.return_value = [ + { + 'id': 10974, + 'name': 'rhscl-3.0-rhel-6-candidate', + }, + { + 'id': 11030, + 'name': 'rhscl-3.0-rhel-6-pending', + }, + { + 'id': 11425, + 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', + } + ] + session.listTagged.return_value = [ + { + 'build_id': 568228, + 'nvr': 'rh-postgresql96-3.0-9.el6', + } + ] + + handler = ErrataAdvisoryRPMsSignedHandler() + tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') + self.assertEqual('rhscl-3.0-rhel-6-candidate', tag) + + @patch('freshmaker.kojiservice.KojiService.session', callable=PropertyMock) + def test_get_None_if_tag_has_new_build(self, session): + session.listTags.return_value = [ + { + 'id': 10974, + 'name': 'rhscl-3.0-rhel-6-candidate', + }, + { + 'id': 11030, + 'name': 'rhscl-3.0-rhel-6-pending', + }, + { + 'id': 11425, + 'name': 'rhscl-3.0-rhel-6-alpha-1.0-set', + } + ] + session.listTagged.return_value = [ + { + 'build_id': 568228, + 'nvr': 'rh-postgresql96-3.0-10.el6', + } + ] + + handler = ErrataAdvisoryRPMsSignedHandler() + tag = handler._get_compose_source('rh-postgresql96-3.0-9.el6') + self.assertEqual(None, tag) + + +class TestPrepareYumRepo(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler._prepare_yum_repo""" + + @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.ODCS') + @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.' + 'ErrataAdvisoryRPMsSignedHandler._get_packages_for_compose') + @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.' + 'ErrataAdvisoryRPMsSignedHandler._get_compose_source') + @patch('time.sleep') + @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.Errata') + def test_get_repo_url_when_succeed_to_generate_compose( + self, errata, sleep, _get_compose_source, + _get_packages_for_compose, ODCS): + _get_packages_for_compose.return_value = ['httpd', 'httpd-debuginfo'] + _get_compose_source.return_value = 'rhel-7.2-candidate' + ODCS.return_value.new_compose.return_value = { + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "source": "f26", + "source_type": 1, + "state": 0, + "state_name": "wait", + } + ODCS.return_value.get_compose.return_value = { + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "source": "f26", + "source_type": 1, + "state": 2, + "state_name": "done", + } + + errata.return_value.get_builds.return_value = set(["httpd-2.4.15-1.f27"]) + + event = Mock(search_key=12345) + handler = ErrataAdvisoryRPMsSignedHandler() + repo_url = handler._prepare_yum_repo(event) + + _get_compose_source.assert_called_once_with("httpd-2.4.15-1.f27") + _get_packages_for_compose.assert_called_once_with("httpd-2.4.15-1.f27") + + # Ensure new_compose is called to request a new compose + ODCS.return_value.new_compose.assert_called_once_with( + 'rhel-7.2-candidate', 'tag', packages=['httpd', 'httpd-debuginfo']) + + # Ensure get_compose is called once in order to get lates state and see + # if it still needs to wait for ODCS + ODCS.return_value.get_compose.assert_called_once_with(3) + + # We should get the right repo URL eventually + self.assertEqual( + 'http://localhost/composes/latest-odcs-3-1/compose/Temporary', + repo_url) + + +class TestFindEventsToInclude(unittest.TestCase): + """Test ErrataAdvisoryRPMsSignedHandler._find_events_to_include""" + + def setUp(self): + db.session.remove() + db.drop_all() + db.create_all() + db.session.commit() + + self.db_event = Event.get_or_create( + db.session, "msg1", "current_event", ErrataAdvisoryRPMsSignedEvent, + released=False) + ArtifactBuild.create(db.session, self.db_event, "foo", "image", 0) + + # Only this event should be reused, because it is unreleased and + # contains the foo build. + ev = Event.get_or_create( + db.session, "msg2", "old_event_foo", ErrataAdvisoryRPMsSignedEvent, + released=False) + ArtifactBuild.create(db.session, ev, "foo", "image", 0) + + ev = Event.get_or_create( + db.session, "msg3", "old_event_foo_released", + ErrataAdvisoryRPMsSignedEvent, released=True) + ArtifactBuild.create(db.session, ev, "foo", "image", 0) + + ev = Event.get_or_create( + db.session, "msg4", "old_event_bar", ErrataAdvisoryRPMsSignedEvent, + released=False) + ArtifactBuild.create(db.session, ev, "bar", "image", 0) + db.session.commit() + + def tearDown(self): + db.session.remove() + db.drop_all() + db.session.commit() + + def test_find_events_to_include(self): + builds = {"foo": MagicMock()} + handler = ErrataAdvisoryRPMsSignedHandler() + events = handler._find_events_to_include(self.db_event, builds) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].search_key, "old_event_foo") diff --git a/tests/test_views.py b/tests/test_views.py index da3fa44..fc365a1 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -173,7 +173,7 @@ class TestViews(unittest.TestCase): self.assertEqual(data['message_id'], '2017-00000000-0000-0000-0000-000000000001') self.assertEqual(data['search_key'], 'RHSA-2018-101') self.assertEqual(data['event_type_id'], models.EVENT_TYPES[events.TestingEvent]) - self.assertEqual(data['builds'], [1, 2, 3]) + self.assertEqual(len(data['builds']), 3) def test_query_events(self): resp = self.client.get('/freshmaker/1/events/')