From 0026104351e5c9026c8fe6dd25880427caaa6610 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Aug 07 2019 22:36:49 +0000 Subject: [PATCH 1/2] Multiple changes to make pylint happy (inc. fake mode changes) This is a large bunch of changes to resolve pylint issues (score goes from mid-5s to high-9s). For some of the issues pylint reported I went beyond just cleanups. Some apparently unused stuff was removed entirely, notably `TestSuite`'s results dict and `store_results` method, and its `in_output` method. The 'fake' environment handling was changed quite a bit to be substantially simpler. This involves changing the way it's invoked - there aren't two args any more, just a single arg for the release version; if it's set to a non-zero value fake mode is enabled, otherwise it isn't. This also removes the ability to run multiple scenarios including 'allinstall' and pass `--releasever` but not `--dryrun`, which would make the other scenarios run in 'real' mode but 'allinstall' run in fake mode. That just seemed like too much of a corner case to bother with; instead we just bail out if you try to include 'allinstall' but don't pass releasever. The fake root cleaning method is changed to a function as it doesn't need to be a method. Ditto for the stuff that creates the fake 'dnf module list' output, there's no need for that to be a class at all. Various methods that returned values were changed not to do so, when the thing that called them doesn't use the return values anyway. Lots of places were using format strings in logger calls, which you're not supposed to do. You're only supposed to use %-style formatting in logger calls. A pain, I know. There's also a lot of unused variables nuked, whitespace fixes, indentation fixes, comparisons like "if foo != None" improved, stuff like that. Signed-off-by: Adam Williamson --- diff --git a/modular_functions.py b/modular_functions.py index fc9cd44..8a6a084 100755 --- a/modular_functions.py +++ b/modular_functions.py @@ -8,186 +8,202 @@ Created on Thu Aug 30 14:30:00 2018 This script uses the dnf to do various operations on modular packages and reports such operations -were succesfull from the users' perspective. It does not test the sanity of the modular content, +were succesful from the users' perspective. It does not test the sanity of the modular content, or any correctness of installed or removed RPM packages. @licence: GPLv3 """ +# stdlib imports import argparse -import dnf import json -import libdnf import logging import os -import pprint -import re +import shutil import subprocess import sys + +# external imports +import dnf +import libdnf import yaml -class DNFoutput: - # This class uses the DNF code to communicate with DNF in order to get a correct and parsed list of all modules to further processing. - def __init__(self): - self.base = dnf.Base() - self.base.read_all_repos() - self.base.fill_sack() - self.modstate = libdnf.module.ModulePackageContainer.ModuleState_UNKNOWN - self.mods = self.base._moduleContainer.getModulePackages() - self.latest = self.base._moduleContainer.getLatestModulesPerRepo(self.modstate, self.mods) - - def _profile_report_formater(self, modulePackage, default_profiles, enabled_str): - installed_profiles = self.base._moduleContainer.getInstalledProfiles(modulePackage.getName()) - available_profiles = modulePackage.getProfiles() - profiles_str = "" - for profile in available_profiles: - profiles_str += "{}{}".format( - profile.getName(), " [d]" if profile.getName() in default_profiles else "") - profiles_str += " [i], " if profile.getName() in installed_profiles and enabled_str \ - else ", " - return profiles_str[:-2] - - def _module_strs_formater(self, modulePackage, markActive=False): - default_str = "" - enabled_str = "" - disabled_str = "" - if modulePackage.getStream() == self.base._moduleContainer.getDefaultStream(modulePackage.getName()): - default_str = " [d]" - if self.base._moduleContainer.isEnabled(modulePackage): - if not default_str: - enabled_str = " " - enabled_str += "[e]" - elif self.base._moduleContainer.isDisabled(modulePackage): - if not default_str: - disabled_str = " " - disabled_str += "[x]" - if markActive and self.base._moduleContainer.isModuleActive(modulePackage): - if not default_str: - disabled_str = " " - disabled_str += "[a]" - return default_str, enabled_str, disabled_str - - def _create_and_fill_table(self, latest): - modules = [] - for latest_per_repo in latest: - for nameStreamArch in latest_per_repo: - if len(nameStreamArch) == 1: - modulePackage = nameStreamArch[0] +def clean_fakeroot(): + """Cleans installation data in fake environment.""" + # All fake tests run in installroot=/testinstall, before each test, the installroot + # should be deleted to achieve a clean test environment. + if os.path.isdir('/testinstall'): + shutil.rmtree('/testinstall') + print("\n======== The test environment has been cleaned. ========\n") + logging.info("The test environment has been wiped succesfully.") + else: + print("\n======== The test environment was not cleaned because it had already " + "been done or did not yet exist. ========\n") + +def _profile_report_formatter(modcon, module_package, default_profiles, enabled_str): + """Generates profile information strings for get_dnf_module_output.""" + installed_profiles = modcon.getInstalledProfiles(module_package.getName()) + available_profiles = module_package.getProfiles() + profiles_str = "" + for profile in available_profiles: + profiles_str += "{}{}".format( + profile.getName(), " [d]" if profile.getName() in default_profiles else "") + profiles_str += " [i], " if profile.getName() in installed_profiles and enabled_str \ + else ", " + return profiles_str[:-2] + +def _module_strs_formatter(modcon, module_package, mark_active=False): + """Generates module information strings for get_dnf_module_output.""" + default_str = "" + enabled_str = "" + disabled_str = "" + if module_package.getStream() == modcon.getDefaultStream(module_package.getName()): + default_str = " [d]" + if modcon.isEnabled(module_package): + if not default_str: + enabled_str = " " + enabled_str += "[e]" + elif modcon.isDisabled(module_package): + if not default_str: + disabled_str = " " + disabled_str += "[x]" + if mark_active and modcon.isModuleActive(module_package): + if not default_str: + disabled_str = " " + disabled_str += "[a]" + return default_str, enabled_str, disabled_str + +def get_dnf_module_output(): + """Uses the DNF library to communicate with DNF in order to get + a correct and parsed list of all modules for further processing. + """ + base = dnf.Base() + base.read_all_repos() + base.fill_sack() + modcon = base._moduleContainer + latest = modcon.getLatestModulesPerRepo( + libdnf.module.ModulePackageContainer.ModuleState_UNKNOWN, + modcon.getModulePackages()) + + modules = [] + for latest_per_repo in latest: + for name_stream_arch in latest_per_repo: + if len(name_stream_arch) == 1: + module_package = name_stream_arch[0] + else: + active = [module for module in name_stream_arch + if modcon.isModuleActive(module)] + if active: + module_package = active[0] else: - active = [module for module in nameStreamArch - if self.base._moduleContainer.isModuleActive(module)] - if active: - modulePackage = active[0] - else: - modulePackage = nameStreamArch[0] - - default_str, enabled_str, disabled_str = self._module_strs_formater(modulePackage) - default_profiles = self.base._moduleContainer.getDefaultProfiles(modulePackage.getName(), modulePackage.getStream()) - profiles_str = self._profile_report_formater(modulePackage, default_profiles, enabled_str) - modules.append( - { - "name": modulePackage.getName(), - "stream": modulePackage.getStream() + default_str + enabled_str + disabled_str, - "profiles": profiles_str, - "summary": modulePackage.getSummary() - } - ) - - return modules + module_package = name_stream_arch[0] + + default_str, enabled_str, disabled_str = _module_strs_formatter(modcon, module_package) + default_profiles = modcon.getDefaultProfiles( + module_package.getName(), module_package.getStream()) + profiles_str = _profile_report_formatter( + modcon, module_package, default_profiles, enabled_str) + modules.append( + { + "name": module_package.getName(), + "stream": module_package.getStream() + default_str + enabled_str + disabled_str, + "profiles": profiles_str, + "summary": module_package.getSummary() + } + ) - def output(self): - return self._create_and_fill_table(self.latest) + return modules class TestSuite: - def __init__(self, dnf, whitelist=None, test='all'): - self.dnf = dnf - self.test = test + """The test suite, which handles running commands and a few other + things for the tests in ModuleTest. + """ + def __init__(self, whitelist=None, releasever=0): self.whitelist = whitelist - self.modlist = {} - self.results = {} + self._module_list = {} self.outputs = {} + if releasever: + print("======== The test will run in fake environment.========") + logging.info("Fake environment has been required for this test.") + clean_fakeroot() + self.extradnfargs = [ + "--installroot=/testinstall", f"--releasever={releasever}", + f"--setopt=module_platform_id=platform:f{releasever}", + "--setopt=tsflags=justdb" + ] + else: + print("======== The test will run on the real host packages.========") + logging.info("Test is running with real system data.") + self.extradnfargs = [] logging.info('The test suite has been initialized.') - self.releasever = None - self.fake = False - self.clean_installation() self.yamldb = {} - #self.install = "--installroot=/testinstall" - #self.relver = f"--releasever={self.releasever}" - #self.extras1 = f"--setopt=module_platform_id=platform:f{self.releasever}" - #self.extras2 = f"--setopt=tsflags=justdb" - - def store_results(self, key, value): - if key in self.results.keys(): - v = self.results[key] - v.append(value) - self.results[key] = v - - else: - self.results[key] = [value] - logging.debug('The store_results method has stored the results into self.results.') + @property def module_list(self): - # Get the modular data from the DNF. It comes as a list of dictionaries. - processed = self.dnf.output() - - # Let us parse the data and for each module create a dictionary, where name is the key and the rest comes as a value. - modulelist = {} - for line in processed: - module = {} - name = line['name'] - module['name'] = name - stream = line['stream'] - # If the stream is a default stream, we can find a '[d]' symbol in it. - # We record the default stream elsewhere and delete the symbol. - if '[d]' in stream: - stream = stream.split('[')[0].strip() - module['default-stream'] = stream - elif '[' in stream: - stream = stream.split('[')[0].strip() - module['stream'] = stream - - # There might be more profiles and one can be set as a default profile. - # This procedure breaks the list of profiles and finds a default one if present. - profiles = line['profiles'].split(',') - nprofiles = [] - for profile in profiles: - if '[d]' in profile: - profile = profile.split('[')[0].strip() - module['default-profile'] = profile - nprofiles.append(profile) - elif '[i]' in profile: - profile = profile.split('[')[0].strip() - nprofiles.append(profile) - else: - nprofiles.append(profile) - module['profiles'] = nprofiles - info = line['summary'] - module['info'] = info - - # Now, let us make a database (dict) with all the modules, with name as keys and list of dicts - # for various streams as values. - if name not in modulelist.keys(): #If this is first occurence - modulelist[name] = [module] - else: # If this is another occurence of same module with different streams. - if module not in modulelist[name]: - modulelist[name].append(module) - else: - pass - - self.modlist = modulelist - - logging.debug('The "dnf module list" operation was succesful. Data have been obtained from the DNF.') - return self.modlist - - def module_info(self, moduleName): + """A dictionary with module names as keys. Values are lists of + dicts, each dict represents an occurrence of the module in the + DNF module list output for a different stream, with stream, + profile and summary information. + """ + if not self._module_list: + # Get the module list from dnf library as a list of dictionaries + # that approximate 'dnf module list' output. + processed = get_dnf_module_output() + + # Let us parse the data and for each module create a dictionary where name is the key + # and the rest comes as a value. + for line in processed: + module = {} + name = line['name'] + module['name'] = name + stream = line['stream'] + # If the stream is a default stream, we can find a '[d]' symbol in it. + # We record the default stream elsewhere and delete the symbol. + if '[d]' in stream: + stream = stream.split('[')[0].strip() + module['default-stream'] = stream + elif '[' in stream: + stream = stream.split('[')[0].strip() + module['stream'] = stream + + # There might be more profiles and one can be set as a default profile. + # This procedure breaks the list of profiles and finds a default one if present. + profiles = line['profiles'].split(',') + nprofiles = [] + for profile in profiles: + if '[d]' in profile: + profile = profile.split('[')[0].strip() + module['default-profile'] = profile + nprofiles.append(profile) + elif '[i]' in profile: + profile = profile.split('[')[0].strip() + nprofiles.append(profile) + else: + nprofiles.append(profile) + module['profiles'] = nprofiles + info = line['summary'] + module['info'] = info + + # Now, let us make a database (dict) with all the modules, with name as keys and + # list of dicts for various streams as values. + if name not in self._module_list: + # This is the first occurrence + self._module_list[name] = [module] + elif module not in self._module_list[name]: + # This is another occurrence of same module with different streams + self._module_list[name].append(module) + logging.debug('DNF module list parsing successful.') + + return self._module_list + + def module_info(self, modname): """Returns a dictionary with streams as keys and profiles as values.""" - modules = self.module_list() - module = modules[moduleName] + module = self.module_list[modname] info = {} - for s in module: - info[s['stream']] = s['profiles'] + for item in module: + info[item['stream']] = item['profiles'] logging.debug("The module_info method ran successfully.") return info @@ -195,57 +211,34 @@ class TestSuite: """This method calls different modular commands.""" # Each time, we need to call a modular dnf command, we will call this method # with different arguments. - if stream == None: - smodifier = '' - else: + smodifier = '' + pmodifier = '' + if stream: smodifier = f":{stream}" - if profile == None: - pmodifier = '' - else: + if profile: pmodifier = f"/{profile}" modifier = f"{module}{smodifier}{pmodifier}" key = f"dnf module {operation} {modifier}" # For some tests, it is good, when we use a fake environment via installroot and we limit - # the DNF operations to "justdb" (only make changes in the database without installing anything). - # This is very useful in case of multiple operations that are much faster. To use this fake option - # do not forget to set_fake(True) before the process. - if self.fake == True: - releasever = self.releasever - install = self.install - relver = self.relver - extras1 = self.extras1 - extras2 = self.extras2 - raw = subprocess.run(['dnf', 'module', operation, '-y', modifier, install, relver, extras1, extras2], capture_output=True) - logging.debug(raw.stdout.decode('utf-8')) - else: - # When the command should be real, we will use a different command. This will make changes to the host system. - raw = subprocess.run(['dnf', 'module', operation, '-y', modifier], capture_output=True) + # the DNF operations to "justdb" (only make changes in the database without installing + # anything). This will be done if 'releasever' is set when initializing the instance, + # it should be set to the desired Fedora release number, e.g. TestSuite(releasever=30) + args = ['dnf', 'module', operation, '-y', modifier] + self.extradnfargs + raw = subprocess.run(args, capture_output=True) + logging.debug(raw.stdout.decode('utf-8')) self.outputs[operation] = raw.stdout.decode('utf-8') if raw.returncode == 0: result = raw.returncode - logging.info(f"The {key} operation ran successfully.") + logging.info("The %s operation ran successfully.", key) result = 0 else: - problem = raw.stderr.decode('utf-8') - logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") + problem = raw.stderr.decode('utf-8') + logging.error("The operation %s has NOT finished successfully, because of %s.", + key, problem) print('There was a DNF problem! Check the log for further information.') result = 1 - self.store_results(f'dnf module {operation}', result) return result - def clean_installation(self): - """Cleans installation data in fake environment.""" - # All fake tests run in installroot=/testinstall, before each test, the installroot - # should be deleted to achieve a clean test environment. - raw = subprocess.run(['rm', '-rf', '/testinstall'], capture_output=True) - if raw.returncode == 0: - print("\n======== The test environment has been cleaned. ========\n") - logging.info("The test environment has been wiped succesfully.") - else: - print("\n======== There was a problem deleting the test environment, because it had been already deleted or never existed. ========\n") - raw = raw.stderr.decode('utf-8') - logging.error(f"The test date have not been wiped succesfully: {raw} ") - def is_listed(self, module, stream, profile, inlist): """Returns if the specified module is listed in dnf module list.""" option = f"--{inlist}" @@ -253,15 +246,8 @@ class TestSuite: # If we operate in the fake environment, we must list in that environment, too, otherwise # we will never get the correct information because the situation is different on the # host computer. - if self.fake == True: - releasever = self.releasever - install = self.install - relver = self.relver - extras1 = self.extras1 - extras2 = self.extras2 - raw = subprocess.run(['dnf', 'module', 'list', option, install, relver, extras1, extras2], capture_output=True) - else: - raw = subprocess.run(['dnf', 'module', 'list', option], capture_output=True) + args = ['dnf', 'module', 'list', option] + self.extradnfargs + raw = subprocess.run(args, capture_output=True) # If the list command ran successfully, parse the list and provide result. if raw.returncode == 0: raw = raw.stdout.decode('utf-8') @@ -273,54 +259,29 @@ class TestSuite: for line in raw: if module in line: modfound = 0 - if stream != None and stream in line: + if not stream or stream in line: strfound = 0 - elif stream == None: - strfound = 0 - if profile != None and profile in line: - profound = 0 - elif profile == None: + if not profile or profile in line: profound = 0 result = modfound + strfound + profound if result == 0: break - # The result show a 1 for every error. Therefore, it can be easily spotted, where the problem is - # even if you do now read the log file. + # The result show a 1 for every error. Therefore, it can be easily spotted, where + # the problem is even if you do not read the log file. # 0 - everything OK # 1 - the specified profile is not in list, but the name and stream are # 10 - the specified stream is not in list, but the name and profile are # 11 - the specified stream and profile are not in list, but the name is # 111 - nothing is listed result = modfound + strfound + profound - logging.info(f"The {key} command ran successfully.") + logging.info("The %s command ran successfully.", key) else: problem = raw.stderr.decode('utf-8') - logging.error(f"The operation {key} has NOT finished successfully, because of {problem}.") + logging.error("The operation %s has NOT finished successfully, because of %s.", + key, problem) result = 1 - self.store_results(key, result) return result - def in_output(self, text, what): - """Checks that the text is in the DNF output.""" - if isinstance(text, str): - text = text.split("\n") - elif isinstance(text, list): - pass - else: - text = [] - c = 0 - for line in text: - if what in line: - c += 1 - else: - pass - if c > 0: - return (1, c) - logging.debug('The string was found in the DNF output.') - else: - return (0, c) - logging.debug('The string was NOT found in the DNF output.') - def load_whitelist(self): """Reads the file with whitelisted modules and returns the content.""" # Some modules do not have the default streams and/or profiles set on purpose. @@ -330,66 +291,52 @@ class TestSuite: else: logging.info("Module whitelist found. It will be used for some tests.") try: - with open(self.whitelist) as f: - wlist = f.readlines() + with open(self.whitelist) as wlfh: + wlist = wlfh.readlines() whitelist = [] - for m in wlist: - whitelist.append(m.strip()) + for module in wlist: + whitelist.append(module.strip()) except FileNotFoundError: - print(f"File {self.whitelist} has not been found. Whitelist could not be read, proceeding without it.") - logging.error(f"File {self.whitelist} has not been found. Proceeding without it.") - whitelist = [] + text = (f"File {self.whitelist} has not been found. Whitelist could not be read, " + "proceeding without it.") + print(text) + logging.error(text) + whitelist = [] return whitelist - def set_fake(self, fake=False, releasever=None): - """Switches the fake environment on.""" - # If you use this in the test method, the operation will run in test environment. Otherwise, it will run - # for real on the host system data. You can also switch the test environment on using -d (--dryrun) when - # invoking the script on CLI. - if fake == 'true' or fake == 'True': - print("======== The test will run in fake environment.========") - logging.info("Fake environment has been required for this test.") - self.fake = True - else: - print("======== The test will run on the real host packages.========") - logging.info("Test is running with real system data.") - self.fake = False - self.install = "--installroot=/testinstall" - self.releasever = releasever - self.relver = f"--releasever={self.releasever}" - self.extras1 = f"--setopt=module_platform_id=platform:f{self.releasever}" - self.extras2 = f"--setopt=tsflags=justdb" - return 0 - def create_yamldb(self, directory): """Read yaml files from the given directory and return a database of collected info.""" listdir = os.listdir(directory) ymls = [yml for yml in listdir if 'yaml' in yml] yamldb = {} for yml in ymls: - f = f"{directory}{yml}" - with open(f,'r') as yamlfile: + filename = f"{directory}{yml}" + with open(filename, 'r') as yamlfile: yamldb[yml] = yaml.safe_load(yamlfile) self.yamldb = yamldb return yamldb def check_yaml_exists(self, module): + """Check yaml file actually exists for a given module.""" key = f"{module}.yaml" try: yamldata = self.yamldb[key] return yamldata except KeyError: print(f"{key}") - logging.error(f"The {key} seems not to exist for {module}.") + logging.error("The %s seems not to exist for %s.", key, module) return None class ModuleTest: - # These are various test methods that do the actual testing. - def __init__(self, suite, scenario): + """This class contains methods that do the actual testing. The + 'overall' dict tracks the overall test results and is updated by + each test method. + """ + def __init__(self, suite, scenarios): self.suite = suite - self.scenario = scenario + self.scenarios = scenarios self.overall = {} self.newer = None self.older = None @@ -402,50 +349,39 @@ class ModuleTest: 110: 'The module stream was not found in the selected list.', 210: 'The module stream and profile were not found in the selected list.', 111: 'The module was not found in the selected list.', - } - - def results(self): - """Return the overall results.""" - return self.overall + } def decode_error(self, code): """Decode the exit error code.""" # We do not use it, yet, I suppose. return self.errorcodes[code] - def list_module(self, module, stream=None, profile=None): + def list_module(self, module): """Provide information on the specified module.""" print('-------- List module ---------') - key = f"{module}:{stream}/{profile}" - modules = self.suite.module_list() - if module in modules.keys(): - info = modules[module] - self.overall['list'] = 'pass' - print(f"Showing information for: {module}\n========================================") - for rec in info: - print(f"Stream: {rec['stream']}") - try: - d = rec['default-stream'] - print(f"Default stream: yes") - except KeyError: - print(f"Default stream: no") - prostring = '' - for p in rec['profiles']: - prostring = f"{prostring} {p}" - print(f"Available profiles: {prostring}") - try: - d = rec['default-profile'] - print(f"Default profile: {d}") - except KeyError: - print(f"Default profile: none") - print('------------------------------') - logging.info("The list method finished successfully.") - return(info) - else: + if module not in self.suite.module_list: print('Module does not exist.') logging.error("Module does not exist.") self.overall['list'] = 'fail' - return None + return + + info = self.suite.module_list[module] + self.overall['list'] = 'pass' + print(f"Showing information for: {module}\n========================================") + for rec in info: + print(f"Stream: {rec['stream']}") + if 'default-stream' in rec: + print("Default stream: yes") + else: + print("Default stream: no") + print("Available profiles: " + ' '.join(rec['profiles'])) + try: + defprof = rec['default-profile'] + print(f"Default profile: {defprof}") + except KeyError: + print(f"Default profile: none") + print('------------------------------') + logging.info("The list method finished successfully.") def check_install(self, module, stream): """Check that a module is installed.""" @@ -454,15 +390,12 @@ class ModuleTest: res1 = self.suite.is_listed(module, stream, 'installed') if res1[key] == 'pass': self.overall['checkinstall'] = 'pass' - logging.info(f"The result of {key} is PASS.") - return 0 else: self.overall['checkinstall'] = 'fail' - logging.info(f"The result of {key} is FAIL.") - return 1 - print(' ') + logging.info("The result of checking %s is %s.", key, self.overall['checkinstall'].upper()) + print('') - def enable_module(self, module, stream, profile = None): + def enable_module(self, module, stream, profile=None): """Test that a module can be enabled.""" print('-------- Enable module ---------') key = f"{module}:{stream}" @@ -477,28 +410,23 @@ class ModuleTest: print(f"{key} is listed in --disabled => {res3}") if res1 == 0 and res2 == 0 and res3 == 111: self.overall['enable'] = 'pass' - logging.info(f"The result of enabling {key} is PASS.") - return 0 else: problems = [res1, res2, res3] - for p in problems: - logging.error(f"Error code = {p}. {self.decode_error(p)}") + for problem in problems: + logging.error("Error code = %s. %s", problem, self.decode_error(problem)) if self.fail == 'hard': self.overall['enable'] = 'fail' - logging.info(f"The result of enabling {key} is FAIL.") - return 1 else: - self.overall['enable'] = 'soft' - logging.info(f"The result of enabling {key} is SOFTFAIL.") - return 2 + self.overall['enable'] = 'softfail' + logging.info("The result of enabling %s is %s.", key, self.overall['enable'].upper()) print('') - def disable_module(self, module, stream, profile = None): + def disable_module(self, module, stream, profile=None): """Test that a module can be disabled.""" print('-------- Disable module ---------') if self.newer != 'dummy': stream = self.newer - if stream != None: + if stream: key = f"{module}:{stream}" else: key = f"{module}" @@ -515,28 +443,23 @@ class ModuleTest: print(f"{key} is listed in --installed =>", res4) if res1 == 0 and res2 == 0 and res3 == 111 and res4 == 111: self.overall['disable'] = 'pass' - logging.info(f"The result of disabling {key} is PASS.") - return 0 else: problems = [res1, res2, res3, res4] - for p in problems: - logging.error(f"Error code = {p}. {self.decode_error(p)}") + for problem in problems: + logging.error("Error code = %s. %s", problem, self.decode_error(problem)) if self.fail == 'hard': self.overall['disable'] = 'fail' - logging.info(f"The result of disabling {key} is FAIL. ") - return 1 else: - self.overall['disable'] = 'soft' - logging.info(f"The result of disabling {key} is SOFTFAIL.") - return 2 + self.overall['disable'] = 'softfail' + logging.info("The result of disabling %s is %s.", key, self.overall['disable'].upper()) print('') def install_module(self, module, stream=None, profile=None): """Test that a module can be installed.""" print('-------- Install module ---------') - if stream != None and profile != None: + if stream and profile: key = f"{module}:{stream}/{profile}" - elif stream != None and profile == None: + elif stream: key = f"{module}:{stream}" else: key = f"{module}" @@ -552,17 +475,12 @@ class ModuleTest: print(f"{key} is listed in --disabled =>", res4) if res1 == 0 and res2 == 0 and res3 == 0 and res4 == 111: self.overall['install'] = 'pass' - logging.info(f"The result of installing {key} is PASS.") - return 0 else: if self.fail == 'hard': self.overall['install'] = 'fail' - logging.info(f"The result of installing {key} is FAIL.") - return 1 else: - self.overall['install'] = 'soft' - logging.info(f"The result of installing {key} is SOFTFAIL.") - return 2 + self.overall['install'] = 'softfail' + logging.info("The result of installing %s is %s.", key, self.overall['install'].upper()) print('') def remove_module(self, module, stream=None, profile=None): @@ -570,7 +488,7 @@ class ModuleTest: print('-------- Remove module ---------') if self.newer != 'dummy': stream = self.newer - if stream != None: + if stream: key = f"{module}:{stream}" else: key = f"{module}" @@ -582,28 +500,23 @@ class ModuleTest: print(f"{key} is listed in --installed =>", res2) if res1 == 0 and res2 == 111: self.overall['remove'] = 'pass' - logging.info(f"The result of removing {key} is PASS.") - return 0 else: if self.fail == 'hard': self.overall['remove'] = 'fail' - logging.info(f"The result of removing {key} is FAIL.") - return 1 else: - self.overall['remove'] = 'soft' - logging.info(f"The result of removing {key} is SOFTFAIL.") - return 2 + self.overall['remove'] = 'softfail' + logging.info("The result of removing %s is %s.", key, self.overall['remove'].upper()) print('') def reset_module(self, module, stream=None, profile=None): """Tests resetting the module.""" # When the module status quo is changed once, it will always play some role in modularity. - # It will either be enabled, disabled, or installed. Removing an installed module, will not disable it, - # disabling an enabled module will not bring it to its neutral state (and vice versa). To bring the - # module into a neutral state, one has to reset it. Resetting a module is an important part of - # module stream switching, if one decides to do so. - # To reset a module, we will check that it is either enabled, or disabled (that means, not neutral, then - # we reset it and check that it is not enabled nor disabled -> neutral. + # It will either be enabled, disabled, or installed. Removing an installed module will + # not disable it; disabling an enabled module will not bring it to its neutral state (and + # vice versa). To bring the module into a neutral state, one has to reset it. Resetting a + # module is an important part of module stream switching, if one decides to do so. + # To reset a module, we will check that it is either enabled, or disabled (that means, not + # neutral), then we reset it and check that it is not enabled nor disabled -> neutral. print('-------- Resetting module ---------') res1 = self.suite.is_listed(module, stream, profile, 'enabled') res2 = self.suite.is_listed(module, stream, profile, 'disabled') @@ -616,12 +529,10 @@ class ModuleTest: print(f"{module} is listed in --disabled => {res2}") if res3 == 0 and res1 == 111 and res2 == 111: self.overall['reset'] = 'pass' - logging.info(f"The result of resetting {module} is PASS.") - return 0 else: self.overall['reset'] = 'fail' - logging.info(f"The result of resetting {module} is FAIL.") - return 1 + logging.info("The result of resetting %s is %s.", + module, self.overall['reset'].upper()) else: print('It seems that the module does not exist or is already reset.') @@ -629,117 +540,115 @@ class ModuleTest: self.overall['reset'] = 'fail' def switch_stream(self, module, oldstr, newstr): - """Tests switching the stream""" - # Direct stream switching is not allowed any more. To switch - # streams, it is required to remove the current module and - # stream, reset it and install the same module with a different - # stream again. This action uses other (predefined) actions to - # achieve it. Same functionality can be achieved by using two - # different tests. + """Tests switching the stream.""" + # Direct stream switching is not allowed any more. To switch streams, it is required to + # remove the current module and stream, reset it and install the same module with a + # different stream again. This action uses other (predefined) actions to achieve it. Same + # functionality can be achieved by using two different tests. print('-------- Switching streams ---------') oldkey = f"{module}:{oldstr}" newkey = f"{module}:{newstr}" - res1 = self.suite.use_module(module, oldstr, 'install') - res2 = self.suite.use_module(module, oldstr, 'remove') - res3 = self.suite.use_module(module, oldstr, 'reset') - res4 = self.suite.use_module(module, newstr, 'install') + res1 = self.suite.use_module(module, oldstr, 'install') + res2 = self.suite.use_module(module, oldstr, 'remove') + res3 = self.suite.use_module(module, oldstr, 'reset') + res4 = self.suite.use_module(module, newstr, 'install') if res1 == 0 and res2 == 0 and res3 == 0 and res4 == 0: self.overall['switch'] = 'pass' - logging.info(f"The result of switching {oldkey} for {newkey} is PASS.") else: if self.fail == 'hard': self.overall['switch'] = 'fail' - logging.info(f"The result of switching {oldkey} for {newkey} is FAIL.") else: - self.overall['switch'] = 'soft' - logging.info(f"The result of switching {oldkey} for {newkey} is SOFTFAIL.") + self.overall['switch'] = 'softfail' + logging.info("The result of switching %s for %s is %s.", oldkey, newkey, + self.overall['switch'].upper()) print('') def find_no_defaults(self): """Finds all modules without default streams and profiles set.""" - modules = self.suite.module_list() - print(f"{len(self.suite.modlist)} modules will be tested.") + modules = self.suite.module_list + print(f"{len(modules)} modules will be tested.") totalresults = {} whitelist = self.suite.load_whitelist() - if len(whitelist)>1: - print(f"{len(whitelist)} whitelisted modules will be skipped. ") - logging.info(f"The following whitelisted modules will be skipped: {whitelist}") - for module in modules.keys(): - varieties = modules[module] + if len(whitelist) > 1: + print(f"{len(whitelist)} whitelisted modules will be skipped.") + logging.info("The following whitelisted modules will be skipped: %s", whitelist) + for (module, varieties) in modules.items(): rstream = [] rprofile = [] - # Originally, this was written to check whether a module has the defaults (stream and profile) - # correctly set. Whitelisted modules were not tested because they did not have the defaults on purpose. - # However, it has been decided that only if there is no default profile set, the test should fail. - # Non default profiles are correct, because the maintainer do not want modular content precede the - # ursine content. + # Originally, this was written to check whether a module has the defaults (stream and + # profile) correctly set. Whitelisted modules were not tested because they did not + # have the defaults on purpose. However, it has been decided that only if there is no + # default profile set, the test should fail. Non default profiles are correct, because + # the maintainer do not want modular content precede the ursine content. for variety in varieties: keys = variety.keys() if 'default-stream' in keys: rstream.append('pass') else: - # When there is no default stream defined, we only should test that this is a wanted - # behaviour by checking the particular yaml file in the defaults git repository. - # Such test will also be added later. Until then, you will be warned to check that - # repository, when a module without default stream is found. + # When there is no default stream defined, we only should test that this is a + # wanted behaviour by checking the particular yaml file in the defaults git + # repository. Such test will also be added later. Until then, you will be + # warned to check that repository, when a module without default stream is + # found. rstream.append('check') if 'default-profile' in keys: rprofile.append('pass') else: - # When no default profile is set, fail immediately, because such module cannot be - # installed using `dnf module install module:stream`, which is a requirement. + # When no default profile is set, fail immediately, because such module cannot + # be installed using `dnf module install module:stream`, which is required. rprofile.append('fail') - logging.error(f"The {module} module has no default profiles set for its streams.") - result = [] + logging.error("The %s module has no default profiles set for its streams.", + module) + results = [] if 'pass' not in rstream: if 'check' in rstream: - result.append('check yaml for stream definition') + results.append('check yaml for stream definition') else: - result.append('stream') + results.append('stream') if 'pass' not in rprofile: - result.append('profile') - if 'stream' not in result and 'profile' not in result: - result.append('pass') + results.append('profile') + if 'stream' not in results and 'profile' not in results: + results.append('pass') if module in whitelist: - result.append('pass') - result.append('whitelisted') + results.append('pass') + results.append('whitelisted') if len(varieties) > 1: - result.append('multi') + results.append('multi') - totalresults[module] = result + totalresults[module] = results logging.info(json.dumps(totalresults, sort_keys=True)) problems = 0 print("======== Showing only problematic modules. ========") - for p in totalresults.keys(): - if 'pass' not in totalresults[p]: - print(f"{p}: {totalresults[p]}") - logging.error(f"{p}: {totalresults[p]}") + for (key, results) in totalresults.items(): + if 'pass' not in results: + out = f"{key}: {results}" + print(out) + logging.error(out) problems += 1 print(f"\nThere were altogether {problems} incomplete modules.") - if problems == 0: - self.overall['checkdefaults'] = 'pass' - logging.info(f"There were no errors in module stream and profile definitions found.") - return 0 - else: + if problems: self.overall['checkdefaults'] = 'fail' - logging.info(f"There were {problems} problems found in module stream and profile definitions.") - return 1 + logging.info("There were %d problems found in module stream and profile definitions.", + problems) + else: + self.overall['checkdefaults'] = 'pass' + logging.info("There were no errors in module stream and profile definitions found.") + + return problems def install_all(self): """Install all modules, all their streams and profiles. One after another.""" - # This tests all available modules, if they can be installed. As this uses - # the fake environment which is newly prepared before each module is - # installed, it downloads all the dnf metadata and therefore takes - # quite a long time. - modules = self.suite.module_list() - self.suite.fake = True - total = len(self.suite.modlist) + # This tests all available modules, if they can be installed. As this uses the fake + # environment which is newly prepared before each module is installed, it downloads all + # the dnf metadata and therefore takes quite a long time. + modules = self.suite.module_list + total = len(modules) if total == 0: total = 1 partial = 0 @@ -748,10 +657,8 @@ class ModuleTest: whitelist = self.suite.load_whitelist() for name in modules.keys(): completed = round((partial/total)*100) - variants = self.suite.module_info(name) - streams = variants.keys() - for stream in streams: - profiles = variants[stream] + variants = self.suite.module_info(name) + for (stream, profiles) in variants.items(): for profile in profiles: profile = profile.strip() print("--------------------------------------------------") @@ -759,10 +666,10 @@ class ModuleTest: if name != prevname: prevname = name partial += 1 - self.suite.clean_installation() - retcode = self.install_module(name, stream, profile) + clean_fakeroot() + self.install_module(name, stream, profile) key = f"{name}:{stream}/{profile}" - if retcode == 0: + if self.overall['install'] == 'pass': self.overall[key] = 'pass' elif name in whitelist: pass @@ -774,13 +681,12 @@ class ModuleTest: # This makes sure that there is yaml file defined for every module in the system defined # and published in https://pagure.io/releng/fedora-module-defaults.git, and that the yaml # files do have all their requirements set. - modules = self.suite.module_list() self.suite.create_yamldb('./yaml/') whitelist = self.suite.load_whitelist() print("------- Non-existing yaml files --------") results = [] parsed_results = {} - for module in modules.keys(): + for module in self.suite.module_list.keys(): parsed = {'pass':[], 'fail':[]} yamldata = self.suite.check_yaml_exists(module) if yamldata: @@ -810,8 +716,8 @@ class ModuleTest: parsed['pass'].append('module') else: parsed['fail'].append('module') - print("The module field value does not match the module name." ) - logging.error("The module field value does not match the module name." ) + print("The module field value does not match the module name.") + logging.error("The module field value does not match the module name.") if modata['profiles']: profiles = modata['profiles'] for key in profiles.keys(): @@ -825,7 +731,7 @@ class ModuleTest: parsed['pass'].append(f"default-stream-{defstr}") else: parsed['fail'].append(f"default-stream-{defstr}") - if len(parsed['fail']) > 0 and module not in whitelist: + if parsed['fail'] and module not in whitelist: parsed_results[module] = parsed['fail'] else: @@ -833,8 +739,8 @@ class ModuleTest: results.append('fail') if parsed_results: - print(f"There are problems with some of the yaml files. Check logs.") - logging.error(f"Problems with yaml files found.") + print("There are problems with some of the yaml files. Check logs.") + logging.error("Problems with yaml files found.") self.overall['yaml_sanity'] = 'fail' else: self.overall['yaml_sanity'] = 'pass' @@ -845,104 +751,93 @@ class ModuleTest: self.overall['yaml_exists'] = 'pass' def run_test(self, module, fail, newer, stream, profile): - """Run a particular test.""" - # This runs the test assigned to an action. An action, can be selected on the CLI - # using the -a (--action) switch. Multiple actions can be specified, for example: - # -a enable,install,remove,disable,reset + """This runs the test assigned to an action. An action can be + selected using the -a (--action) switch. Multiple actions can + be specified, for example: + -a enable,install,remove,disable,reset + """ self.newer = newer self.older = stream self.fail = fail - if stream != None: + if stream: key = f"{module}:{stream}" else: key = f"{module}" - print(f"\n======= Starting tests for {key} using {self.scenario} =======\n") - for s in self.scenario: - if s == 'enable': + print(f"\n======= Starting tests for {key} using {self.scenarios} =======\n") + for scenario in self.scenarios: + if scenario == 'enable': self.enable_module(module, stream) - elif s == 'disable': + elif scenario == 'disable': self.disable_module(module, stream) - elif s == 'install': + elif scenario == 'install': self.install_module(module, stream, profile) - elif s == 'remove': + elif scenario == 'remove': self.remove_module(module, stream) - elif s == 'switch': + elif scenario == 'switch': self.switch_stream(module, stream, newer) - elif s == 'list': - self.list_module(module, stream) - elif s == 'checkinstall': + elif scenario == 'list': + self.list_module(module) + elif scenario == 'checkinstall': self.check_install(module, stream) - elif s == 'reset': + elif scenario == 'reset': self.reset_module(module) - elif s == 'info': + elif scenario == 'info': info = self.suite.module_info(module) print(f"Available streams and profiles of the {module} module:") print("----------------------------------") print(json.dumps(info, sort_keys=True, indent=4)) - elif s == 'checkdefaults': + elif scenario == 'checkdefaults': self.find_no_defaults() - elif s == 'testyaml': + elif scenario == 'testyaml': self.testyaml() - elif s == 'allinstall': + elif scenario == 'allinstall': self.install_all() - -class Parser: - def __init__(self): - """Inititate the CLI control.""" - self.parser = argparse.ArgumentParser() - self.parser.add_argument('-m', '--module', default='testmodule', help='The name of the module you wish to work with.') - self.parser.add_argument('-s', '--stream', default=None, help='The name of the stream you want to use.') - self.parser.add_argument('-u', '--upgrade', default='dummy', help='The name of the stream you want to switch to.') - self.parser.add_argument('-a', '--action',default='list', help='List of actions, you want to run.') - self.parser.add_argument('-f', '--fail',default='hard', help='Fail hard/soft at list errors.') - self.parser.add_argument('-l', '--log', default='info', help='Log Level use (info, warning, debug).') - self.parser.add_argument('-p', '--profile', default=None, help='The name of the profile you want to use.') - self.parser.add_argument('-w', '--whitelist', default=None, help='File with modules to be skipped when testing default streams and profiles.') - self.parser.add_argument('-r', '--releasever', default=None, help='Sets the release version number for fake installation test (use only when you want to invoke mass instalation).') - self.parser.add_argument('-d', '--dryrun', default=False, help='Switches dry run when installing modules.') - #self.parser.add_argument('--everything', default=False, help='Run all tests on all modules in the system.') - - def return_args(self): - args = self.parser.parse_args() - return args - - -if __name__ == '__main__': - options = Parser() - args = options.return_args() - args = args.__dict__ - module = args['module'] - oldstream = args['stream'] - profile = args['profile'] - newstream = args['upgrade'] - action = args['action'].split(',') - fail = args['fail'] - log = args['log'] - whitelist = args['whitelist'] - releasever = args['releasever'] - #everything = args['everything'] - fake = args['dryrun'] - numloglevel = getattr(logging, log.upper()) +def parse_args(): + """Parse and return command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--module', default='testmodule', + help='The name of the module you wish to work with.') + parser.add_argument('-s', '--stream', default=None, + help='The name of the stream you want to use.') + parser.add_argument('-u', '--upgrade', default='dummy', + help='The name of the stream you want to switch to.') + parser.add_argument('-a', '--action', default='list', help='List of actions, you want to run.') + parser.add_argument('-f', '--fail', default='hard', help='Fail hard/soft at list errors.') + parser.add_argument('-l', '--log', default='info', + help='Log Level use (info, warning, debug).') + parser.add_argument('-p', '--profile', default=None, + help='The name of the profile you want to use.') + parser.add_argument('-w', '--whitelist', default=None, help='File with modules to be skipped ' + 'when testing default streams and profiles.') + parser.add_argument('-r', '--releasever', '-d', '--dryrun', type=int, default=0, + help='A Fedora release number: if set, a non-default install root will be ' + 'used so as not to affect the installed system.') + #parser.add_argument('--everything', default=False, + # help='Run all tests on all modules in the system.') + + return parser.parse_args() + +def main(): + """Do the things! Print the output!""" + args = parse_args() + action = args.action.split(',') + if 'allinstall' in action and not args.releasever: + sys.exit("allinstall action must be run in fake install root, pass --releasever!") + numloglevel = getattr(logging, args.log.upper()) logging.basicConfig(filename='modular.log', filemode='w', level=numloglevel) logging.info('Script started.') - dnf = DNFoutput() - suite = TestSuite(dnf, whitelist) - # When releasever and fake are not set, it will send the False value - # and the fake environment will not be switched on. - # Release version is important for the installroot, it will not install anything without it. - # The correct releasver should be taked from environmental variables, if you want to use - # this script in OpenQA or some others CIs. - suite.set_fake(fake, releasever) + # When releasever is set, the fake install root will be enabled + suite = TestSuite(whitelist=args.whitelist, releasever=args.releasever) test = ModuleTest(suite, action) - test.run_test(module, fail, newstream, oldstream, profile) + test.run_test(args.module, args.fail, args.upgrade, args.stream, args.profile) print('') print('========= Results ==========') - results = test.results() + results = test.overall for key in results: print(f"Tested functionality: {key} => {results[key]}") suite.outputs['total_results'] = results @@ -954,3 +849,10 @@ if __name__ == '__main__': else: logging.info('Script finished with exit code 0.') sys.exit(0) + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.stderr.write("Interrupted, exiting...\n") + sys.exit(1) From 3234fc83a6877895606b0ab2f2f1cb8d71b60ef5 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Aug 08 2019 00:34:58 +0000 Subject: [PATCH 2/2] Construct TestSuite.module_list much more efficiently So this used to get constructed by literally running 'dnf module list' and then parsing the output. Then it got re-written to use the dnf library to quite laboriously construct a thing which looked a lot *like* the output of 'dnf module list' only without some awkward-to-parse bits, and run *that* through the existing parser to get the dict-of-lists-of-dicts that we ultimately want. Take a step back and that is rather silly, though. We're doing a bunch of fragile string concatenation to produce a whole intermediate format we don't really need and then doing fragile string parsing on that to produce the thing we *do* want. Let's just cut out the middleman and have `module_list` build itself directly from the info we get from the dnf library. This makes the whole thing smaller, more robust and 100% less reliant on string parsing... Signed-off-by: Adam Williamson --- diff --git a/modular_functions.py b/modular_functions.py index 8a6a084..921b219 100755 --- a/modular_functions.py +++ b/modular_functions.py @@ -41,80 +41,6 @@ def clean_fakeroot(): print("\n======== The test environment was not cleaned because it had already " "been done or did not yet exist. ========\n") -def _profile_report_formatter(modcon, module_package, default_profiles, enabled_str): - """Generates profile information strings for get_dnf_module_output.""" - installed_profiles = modcon.getInstalledProfiles(module_package.getName()) - available_profiles = module_package.getProfiles() - profiles_str = "" - for profile in available_profiles: - profiles_str += "{}{}".format( - profile.getName(), " [d]" if profile.getName() in default_profiles else "") - profiles_str += " [i], " if profile.getName() in installed_profiles and enabled_str \ - else ", " - return profiles_str[:-2] - -def _module_strs_formatter(modcon, module_package, mark_active=False): - """Generates module information strings for get_dnf_module_output.""" - default_str = "" - enabled_str = "" - disabled_str = "" - if module_package.getStream() == modcon.getDefaultStream(module_package.getName()): - default_str = " [d]" - if modcon.isEnabled(module_package): - if not default_str: - enabled_str = " " - enabled_str += "[e]" - elif modcon.isDisabled(module_package): - if not default_str: - disabled_str = " " - disabled_str += "[x]" - if mark_active and modcon.isModuleActive(module_package): - if not default_str: - disabled_str = " " - disabled_str += "[a]" - return default_str, enabled_str, disabled_str - -def get_dnf_module_output(): - """Uses the DNF library to communicate with DNF in order to get - a correct and parsed list of all modules for further processing. - """ - base = dnf.Base() - base.read_all_repos() - base.fill_sack() - modcon = base._moduleContainer - latest = modcon.getLatestModulesPerRepo( - libdnf.module.ModulePackageContainer.ModuleState_UNKNOWN, - modcon.getModulePackages()) - - modules = [] - for latest_per_repo in latest: - for name_stream_arch in latest_per_repo: - if len(name_stream_arch) == 1: - module_package = name_stream_arch[0] - else: - active = [module for module in name_stream_arch - if modcon.isModuleActive(module)] - if active: - module_package = active[0] - else: - module_package = name_stream_arch[0] - - default_str, enabled_str, disabled_str = _module_strs_formatter(modcon, module_package) - default_profiles = modcon.getDefaultProfiles( - module_package.getName(), module_package.getStream()) - profiles_str = _profile_report_formatter( - modcon, module_package, default_profiles, enabled_str) - modules.append( - { - "name": module_package.getName(), - "stream": module_package.getStream() + default_str + enabled_str + disabled_str, - "profiles": profiles_str, - "summary": module_package.getSummary() - } - ) - - return modules - class TestSuite: """The test suite, which handles running commands and a few other @@ -148,52 +74,54 @@ class TestSuite: profile and summary information. """ if not self._module_list: - # Get the module list from dnf library as a list of dictionaries - # that approximate 'dnf module list' output. - processed = get_dnf_module_output() - - # Let us parse the data and for each module create a dictionary where name is the key - # and the rest comes as a value. - for line in processed: - module = {} - name = line['name'] - module['name'] = name - stream = line['stream'] - # If the stream is a default stream, we can find a '[d]' symbol in it. - # We record the default stream elsewhere and delete the symbol. - if '[d]' in stream: - stream = stream.split('[')[0].strip() - module['default-stream'] = stream - elif '[' in stream: - stream = stream.split('[')[0].strip() - module['stream'] = stream - - # There might be more profiles and one can be set as a default profile. - # This procedure breaks the list of profiles and finds a default one if present. - profiles = line['profiles'].split(',') - nprofiles = [] - for profile in profiles: - if '[d]' in profile: - profile = profile.split('[')[0].strip() - module['default-profile'] = profile - nprofiles.append(profile) - elif '[i]' in profile: - profile = profile.split('[')[0].strip() - nprofiles.append(profile) + base = dnf.Base() + base.read_all_repos() + base.fill_sack() + modcon = base._moduleContainer + latest = modcon.getLatestModulesPerRepo( + libdnf.module.ModulePackageContainer.ModuleState_UNKNOWN, + modcon.getModulePackages()) + + for latest_per_repo in latest: + for name_stream_arch in latest_per_repo: + if len(name_stream_arch) == 1: + module_package = name_stream_arch[0] else: - nprofiles.append(profile) - module['profiles'] = nprofiles - info = line['summary'] - module['info'] = info - - # Now, let us make a database (dict) with all the modules, with name as keys and - # list of dicts for various streams as values. - if name not in self._module_list: - # This is the first occurrence - self._module_list[name] = [module] - elif module not in self._module_list[name]: - # This is another occurrence of same module with different streams - self._module_list[name].append(module) + active = [module for module in name_stream_arch + if modcon.isModuleActive(module)] + if active: + module_package = active[0] + else: + module_package = name_stream_arch[0] + + name = module_package.getName() + stream = module_package.getStream() + module = { + 'name': name, + 'stream': stream, + 'profiles': [], + 'info': module_package.getSummary() + } + if stream == modcon.getDefaultStream(name): + module['default-stream'] = stream + + default_profiles = modcon.getDefaultProfiles(name, stream) + available_profiles = module_package.getProfiles() + for profile in available_profiles: + profname = profile.getName() + module['profiles'].append(profname) + if profname in default_profiles: + module['default-profile'] = profname + + # Now, let us make a database (dict) with all the modules, with name as keys + # and list of dicts for various streams as values. + if name not in self._module_list: + # This is the first occurrence + self._module_list[name] = [module] + elif module not in self._module_list[name]: + # This is another occurrence of same module with different streams + self._module_list[name].append(module) + logging.debug('DNF module list parsing successful.') return self._module_list