From e09d0d1e47a215000d05c9a87fc34843543625a6 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 11 2017 11:59:45 +0000 Subject: [PATCH 1/6] improved multihost test handling and created functions for normalizing commands before run (escaping) --- diff --git a/examples/multios_testing/sanity1.py b/examples/multios_testing/sanity1.py index 51f6245..7de8f11 100644 --- a/examples/multios_testing/sanity1.py +++ b/examples/multios_testing/sanity1.py @@ -43,7 +43,7 @@ class SanityCheck1(module_framework.AvocadoTest): time.sleep(2) b = self.run("cat %s" %TFILE).stdout.strip() self.assertEqual(a,b) - self.assertIn("25", b) + self.assertIn("26", b) diff --git a/moduleframework/common.py b/moduleframework/common.py index a01fba0..e547ed7 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -213,4 +213,8 @@ def normalize_text(text, replacement="_"): badchars=["/", ";", "&", ">", "<", "|"] for foo in badchars: out = out.replace(foo, replacement) - return out \ No newline at end of file + return out + +def normalize_cmd(cmd): + cmd = cmd.replace('"', r'\"') + return cmd \ No newline at end of file diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 98639c4..7f6717a 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -73,6 +73,7 @@ class CommonFunctions(object): self.source = None self.arch = None self.dependencylist = {} + self.moduledeps = None # general use case is to have forwarded services to host (so thats why it is same) self.ipaddr = trans_dict["HOSTIPADDR"] trans_dict["GUESTARCH"] = self.getArch() @@ -422,7 +423,7 @@ class ContainerHelper(CommonFunctions): self.start() return self.runHost( 'docker exec %s bash -c "%s"' % - (self.docker_id, command.replace('"', r'\"')), + (self.docker_id, normalize_cmd(command)), **kwargs) def copyTo(self, src, dest): @@ -675,7 +676,7 @@ gpgcheck=0 :return: avocado.process.run """ return self.runHost('bash -c "%s"' % - command.replace('"', r'\"'), **kwargs) + normalize_cmd(command), **kwargs) def copyTo(self, src, dest): """ @@ -758,7 +759,7 @@ class NspawnHelper(RpmHelper): # (failing because of selinux) self.__selinuxState = self.runHost( "getenforce", ignore_status=True).stdout.strip() - self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent()) + self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent(), sudo=True) self.setModuleDependencies() self.setRepositoriesAndWhatToInstall() self.installTestDependencies() @@ -768,7 +769,7 @@ class NspawnHelper(RpmHelper): def __is_killed(self): for foo in range(DEFAULTRETRYTIMEOUT): - time.sleep(foo) + time.sleep(1) out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) if out.exit_status != 0: print_debug("NSPAWN machine %s stopped" % self.jmeno) @@ -777,9 +778,9 @@ class NspawnHelper(RpmHelper): def __is_booted(self): for foo in range(DEFAULTRETRYTIMEOUT): - time.sleep(foo) + time.sleep(1) out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) - if "logind.service" in out.stdout: + if "systemd-logind" in out.stdout: time.sleep(2) print_debug("NSPAWN machine %s booted" % self.jmeno) return True @@ -797,17 +798,12 @@ class NspawnHelper(RpmHelper): if os.path.exists(self.chrootpath): shutil.rmtree(self.chrootpath, ignore_errors=True) # DELETE every chroot dir in case any exists - dirstodelete = glob.glob(self.baseprefix + "*") - if dirstodelete: - for dtd in dirstodelete: - shutil.rmtree(dtd, ignore_errors=True) - # Terminate machine in case of same name and still running - try: - self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug(), ignore_status=True) - self.__is_killed() - except BaseException: - pass - os.mkdir(self.chrootpath) + # Commented out, because it had side effect for multihost testing. Has to be improved + #dirstodelete = glob.glob(self.baseprefix + "*") + #if get_if_module() and dirstodelete: + # for dtd in dirstodelete: + # shutil.rmtree(dtd, ignore_errors=True) + os.mkdir(self.chrootpath) def __prepareSetup(self): """ @@ -817,7 +813,9 @@ class NspawnHelper(RpmHelper): """ self.__do_smart_start_cleanup() if not os.path.exists(os.path.join(self.chrootpath, "usr")): - self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent()) + self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent(), sudo=True) + # workaround in case machined blocked by selinux, disabled for now + # self.runHost("sudo systemctl restart systemd-machined", verbose=is_not_silent(), sudo=True) repos_to_use = "" counter = 0 for repo in self.repos: @@ -958,20 +956,20 @@ gpgcheck=0 :return: avocado.process.run """ lpath = "/var/tmp" - comout = self.runHost( - """machinectl shell root@{machine} /bin/bash -c "({comm})>{pin}/stdout 2>{pin}/stderr; echo $?>{pin}/retcode; sleep 1" """.format( + if not kwargs: + kwargs = {} + should_ignore = kwargs.get("ignore_status") + kwargs["ignore_status"] = True + + comout = self.runHost("""machinectl shell root@{machine} /bin/bash -c "({comm})>{pin}/stdout 2>{pin}/stderr; echo $?>{pin}/retcode; sleep 1" """.format( machine=self.jmeno, - comm=command.replace( - '"', - r'\"'), + comm=normalize_cmd(command), pin=lpath), - **kwargs) + **kwargs) + if comout.exit_status != 0: + raise NspawnExc("This command should not fail anyhow inside NSPAWN:", normalize_cmd(command)) try: - if not kwargs: - kwargs = {} kwargs["verbose"] = is_not_silent() - should_ignore = kwargs.get("ignore_status") - kwargs["ignore_status"] = True b = self.runHost( 'bash -c "cat {chroot}{pin}/stdout; cat {chroot}{pin}/stderr > /dev/stderr; exit `cat {chroot}{pin}/retcode`"'.format( chroot=self.chrootpath, @@ -1040,7 +1038,6 @@ gpgcheck=0 except Exception as poweroffex: print_info("Unable to stop machine via poweroff, terminating", poweroffex) try: - time.sleep(1) self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True) self.__is_killed() except Exception as poweroffexterm: From 134534a8b097afaf5143ee0a7539156531a669a9 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 12 2017 06:14:11 +0000 Subject: [PATCH 2/6] changed return variable to not be same as input param --- diff --git a/moduleframework/common.py b/moduleframework/common.py index e547ed7..9713f45 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -216,5 +216,5 @@ def normalize_text(text, replacement="_"): return out def normalize_cmd(cmd): - cmd = cmd.replace('"', r'\"') - return cmd \ No newline at end of file + command = cmd.replace('"', r'\"') + return command \ No newline at end of file From 973e879a054ec43a071a2f64cff083cde1afb42d Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 12 2017 07:31:04 +0000 Subject: [PATCH 3/6] function names cleanup, removed "correct" "latest" words fixing command sanitizer --- diff --git a/moduleframework/common.py b/moduleframework/common.py index 9713f45..d9e815a 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -202,19 +202,32 @@ def get_if_module(): return not bool(rreps) -def normalize_text(text, replacement="_"): +def sanitize_text(text, replacement="_"): + """ - Improve string, replace all bad characters with another one expecially with "_" + Replace invalid characters in a string. + + invalid_chars=["/", ";", "&", ">", "<", "|"] :param text: string + :param replacement: replacement char, default: "_" :return: string """ - out = text - badchars=["/", ";", "&", ">", "<", "|"] - for foo in badchars: - out = out.replace(foo, replacement) - return out + invalid_chars=["/", ";", "&", ">", "<", "|"] + for char in invalid_chars: + if char in text: + text = text.replace(char, replacement) + return text + +def sanitize_cmd(cmd): + """ + Do escaping of characters for command inside apostrophes -def normalize_cmd(cmd): - command = cmd.replace('"', r'\"') - return command \ No newline at end of file + :param cmd: string + :return: string + """ + escaping_chars = ['"'] + for char in escaping_chars: + if char in cmd: + cmd = cmd.replace(char, '\\'.join(char)) + return cmd diff --git a/moduleframework/compose_info.py b/moduleframework/compose_info.py index cb07ac0..b49bd52 100644 --- a/moduleframework/compose_info.py +++ b/moduleframework/compose_info.py @@ -32,7 +32,6 @@ import urllib import xml.etree.ElementTree import gzip import tempfile -import os from common import * diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 7f6717a..952835d 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -141,7 +141,7 @@ class CommonFunctions(object): """ try: self.config = get_config() - self.moduleName = normalize_text(self.config['name']) + self.moduleName = sanitize_text(self.config['name']) self.source = self.config.get('source') if self.config.get( 'source') else self.config['module']['rpm'].get('source') except ValueError: @@ -167,8 +167,8 @@ class CommonFunctions(object): out += packages_rpm + packages_profiles elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data'][ - 'profiles'].get(get_correct_profile()): - out += self.getModulemdYamlconfig()['data']['profiles'][get_correct_profile()]['rpms'] + 'profiles'].get(get_profile()): + out += self.getModulemdYamlconfig()['data']['profiles'][get_profile()]['rpms'] else: # fallback solution when it is not known what to install out.append("bash") @@ -200,7 +200,7 @@ class CommonFunctions(object): if self.config is None: self.loadconfig() if not self.modulemdConf: - modulemd = get_correct_modulemd() + modulemd = get_modulemd() if modulemd: ymlfile = urllib.urlopen(modulemd) self.modulemdConf = yaml.load(ymlfile) @@ -238,8 +238,8 @@ class ContainerHelper(CommonFunctions): self.tarbased = None self.jmeno = None self.docker_id = None - self.icontainer = get_correct_url( - ) if get_correct_url() else self.info['container'] + self.icontainer = get_url( + ) if get_url() else self.info['container'] if ".tar" in self.icontainer: self.jmeno = "testcontainer" self.tarbased = True @@ -423,7 +423,7 @@ class ContainerHelper(CommonFunctions): self.start() return self.runHost( 'docker exec %s bash -c "%s"' % - (self.docker_id, normalize_cmd(command)), + (self.docker_id, sanitize_cmd(command)), **kwargs) def copyTo(self, src, dest): @@ -553,12 +553,12 @@ class RpmHelper(CommonFunctions): else: if not self.repos: for dep in self.moduledeps: - latesturl = get_latest_repo_url(dep, self.moduledeps[dep]) + latesturl = get_repo_url(dep, self.moduledeps[dep]) alldrepos.append(latesturl) self.__addModuleDependency(url=latesturl, name = dep, stream = self.moduledeps[dep]) - if get_correct_url(): - self.repos = [get_correct_url()] + alldrepos - self.__addModuleDependency(get_correct_url()) + if get_url(): + self.repos = [get_url()] + alldrepos + self.__addModuleDependency(get_url()) elif self.info.get('repo'): self.repos = [self.info.get('repo')] + alldrepos self.__addModuleDependency(self.info.get('repo')) @@ -676,7 +676,7 @@ gpgcheck=0 :return: avocado.process.run """ return self.runHost('bash -c "%s"' % - normalize_cmd(command), **kwargs) + sanitize_cmd(command), **kwargs) def copyTo(self, src, dest): """ @@ -963,11 +963,11 @@ gpgcheck=0 comout = self.runHost("""machinectl shell root@{machine} /bin/bash -c "({comm})>{pin}/stdout 2>{pin}/stderr; echo $?>{pin}/retcode; sleep 1" """.format( machine=self.jmeno, - comm=normalize_cmd(command), + comm=sanitize_cmd(command), pin=lpath), **kwargs) if comout.exit_status != 0: - raise NspawnExc("This command should not fail anyhow inside NSPAWN:", normalize_cmd(command)) + raise NspawnExc("This command should not fail anyhow inside NSPAWN:", sanitize_cmd(command)) try: kwargs["verbose"] = is_not_silent() b = self.runHost( @@ -1094,8 +1094,8 @@ class AvocadoTest(Test): def __init__(self, *args, **kwargs): super(AvocadoTest, self).__init__(*args, **kwargs) - (self.backend, self.moduleType) = get_correct_backend() - self.moduleProfile = get_correct_profile() + (self.backend, self.moduleType) = get_backend() + self.moduleProfile = get_profile() print_info( "Module Type: %s; Profile: %s" % (self.moduleType, self.moduleProfile)) @@ -1336,7 +1336,7 @@ class NspawnAvocadoTest(AvocadoTest): super(NspawnAvocadoTest, self).setUp() -def get_correct_backend(): +def get_backend(): """ Return proper module type, set by config by default_module section, or defined via env variable "MODULE" @@ -1359,7 +1359,7 @@ def get_correct_backend(): raise ModuleFrameworkException("Unsupported MODULE={0}".format(amodule), "supproted are: docker, rpm, nspawn") -def get_correct_profile(): +def get_profile(): """ Return profile name string @@ -1371,7 +1371,7 @@ def get_correct_profile(): return amodule -def get_correct_url(): +def get_url(): """ Return actual URL if overwritten by env variable "URL" @@ -1403,6 +1403,7 @@ def get_config(): cfgfile + " " + "Tip: If the CONFIG envvar is not set, mtf-generator looks for './config'.") + def get_compose_url(): """ Return Compose Url if set in config or via @@ -1423,7 +1424,7 @@ def get_compose_url(): return compose -def get_correct_modulemd(): +def get_modulemd(): """ Return dict of moduleMD file for module, It is read from config, from module-url section, if not defined it reads modulemd file from compose-url in case of set, or there is used @@ -1445,22 +1446,3 @@ def get_correct_modulemd(): return [x[12:] for x in b if 'MODULEMDURL=' in x][0] except AttributeError: return None - - -def get_latest_repo_url(wmodule="base-runtime", wstream="master", fake=False): - """ - Return URL location of rpm repository. - It reads data from PDC and construct url locator. - It is used to solve repos for dependent modules (eg. memcached is dependent on perl and baseruntime) - - :param wmodule: module name - :param wstream: module stream - :param fake: - :return: str - """ - if fake: - return "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/" - else: - tmp_pdc = pdc_data.PDCParser() - tmp_pdc.setLatestPDC(wmodule, wstream) - return tmp_pdc.generateRepoUrl() diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index 354070e..2c07564 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -76,6 +76,24 @@ def getBasePackageSet(modulesDict=None, isModule=True, isContainer=False): print_info("ALL packages to install:", out) return out +def get_repo_url(wmodule="base-runtime", wstream="master", fake=False): + """ + Return URL location of rpm repository. + It reads data from PDC and construct url locator. + It is used to solve repos for dependent modules (eg. memcached is dependent on perl and baseruntime) + + :param wmodule: module name + :param wstream: module stream + :param fake: + :return: str + """ + if fake: + return "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/" + else: + tmp_pdc = PDCParser() + tmp_pdc.setLatestPDC(wmodule, wstream) + return tmp_pdc.generateRepoUrl() + class PDCParser(): """ From 8827b9e387ff8b3021bb6e9504f2b03290b0c2fe Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 12 2017 07:36:56 +0000 Subject: [PATCH 4/6] fixed typo caused by function moving to other module --- diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 952835d..3e721c6 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -553,7 +553,7 @@ class RpmHelper(CommonFunctions): else: if not self.repos: for dep in self.moduledeps: - latesturl = get_repo_url(dep, self.moduledeps[dep]) + latesturl = pdc_data.get_repo_url(dep, self.moduledeps[dep]) alldrepos.append(latesturl) self.__addModuleDependency(url=latesturl, name = dep, stream = self.moduledeps[dep]) if get_url(): From 300b0ba00282c6e0a2448c3d4eb67b26f06b423b Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 12 2017 08:31:00 +0000 Subject: [PATCH 5/6] replaced get_correct_backed in all internal tests by get_backend --- diff --git a/examples/multios_testing/sanityRealMultiHost.py b/examples/multios_testing/sanityRealMultiHost.py index d607e06..f519599 100644 --- a/examples/multios_testing/sanityRealMultiHost.py +++ b/examples/multios_testing/sanityRealMultiHost.py @@ -32,9 +32,9 @@ class SanityRealMultihost(module_framework.AvocadoTest): :avocado: enable """ def setUp(self): - self.machineF25 = module_framework.get_correct_backend()[0] - self.machineF26 = module_framework.get_correct_backend()[0] - self.machineRawhide = module_framework.get_correct_backend()[0] + self.machineF25 = module_framework.get_backend()[0] + self.machineF26 = module_framework.get_backend()[0] + self.machineRawhide = module_framework.get_backend()[0] self.machineF25.setRepositoriesAndWhatToInstall(repos = ["http://ftp.fi.muni.cz/pub/linux/fedora/linux/releases/25/Everything/x86_64/os/"]) self.machineF26.setRepositoriesAndWhatToInstall(repos = ["http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/26/Everything/x86_64/os/"]) self.machineRawhide.setRepositoriesAndWhatToInstall(repos = ["http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/rawhide/Everything/x86_64/os/"]) diff --git a/examples/testing-module/PureAvocadoTest.py b/examples/testing-module/PureAvocadoTest.py index bdf7ac5..e90edee 100644 --- a/examples/testing-module/PureAvocadoTest.py +++ b/examples/testing-module/PureAvocadoTest.py @@ -29,7 +29,7 @@ from avocado import utils class PureAvocadoTest(Test): def setUp(self): - self.backend, self.moduletype = module_framework.get_correct_backend() + self.backend, self.moduletype = module_framework.get_backend() self.backend.setUp() def testInsideModule(self): diff --git a/moduleframework/bashhelper.py b/moduleframework/bashhelper.py index 1abf965..c2f96e7 100755 --- a/moduleframework/bashhelper.py +++ b/moduleframework/bashhelper.py @@ -70,7 +70,7 @@ def main(): "Unable to call bash helper without function, there is possible to use: ", [a[0] for a in inspect.getmembers( - module_framework.get_correct_backend(), + module_framework.get_backend(), predicate=inspect.ismethod) if '__' not in a[0]]) method = args[0] @@ -86,7 +86,7 @@ def main(): printIfVerbose("reading from pickled object", helper) pkl_file.close() else: - (helper, moduletype) = module_framework.get_correct_backend() + (helper, moduletype) = module_framework.get_backend() printIfVerbose("created new instance for module") pkl_file = open(picklefile, 'wb') diff --git a/moduleframework/general_multiplex.yaml b/moduleframework/general_multiplex.yaml deleted file mode 100644 index 40120a7..0000000 --- a/moduleframework/general_multiplex.yaml +++ /dev/null @@ -1,5 +0,0 @@ -module-type: !mux - docker: - module: 'docker' - rpm: - module: 'rpm' diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 3e721c6..440306b 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -1358,6 +1358,9 @@ def get_backend(): else: raise ModuleFrameworkException("Unsupported MODULE={0}".format(amodule), "supproted are: docker, rpm, nspawn") +# To keep backward compatibility. This method could be used by pure avocado tests and is already used +get_correct_backend = get_backend + def get_profile(): """ From 638146dc98d1b09b1c691eab357886b88864e328 Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jul 12 2017 13:05:06 +0000 Subject: [PATCH 6/6] Draft of refactoring avocado-tests. Signed-off-by: Petr "Stone" Hracek --- diff --git a/modularity-testing-framework.spec b/modularity-testing-framework.spec index 31f12a4..7a9e8cd 100644 --- a/modularity-testing-framework.spec +++ b/modularity-testing-framework.spec @@ -22,6 +22,7 @@ Requires: docker Requires: python2-dockerfile-parse Requires: python2-pdc-client Requires: python2-modulemd +Requires: python-retrying %description %{summary}. diff --git a/moduleframework/avocado_testers/__init__.py b/moduleframework/avocado_testers/__init__.py new file mode 100644 index 0000000..cbb80c5 --- /dev/null +++ b/moduleframework/avocado_testers/__init__.py @@ -0,0 +1,22 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# diff --git a/moduleframework/avocado_testers/container_avocado_test.py b/moduleframework/avocado_testers/container_avocado_test.py new file mode 100644 index 0000000..a345817 --- /dev/null +++ b/moduleframework/avocado_testers/container_avocado_test.py @@ -0,0 +1,54 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# + +from moduleframework.module_framework import AvocadoTest + + +# INTERFACE CLASSES FOR SPECIFIC MODULE TESTS +class ContainerAvocadoTest(AvocadoTest): + """ + Class for writing tests specific just for DOCKER + derived from AvocadoTest class. + + :avocado: disable + """ + + def setUp(self): + if self.moduleType != "docker": + self.skip("Docker specific test") + super(ContainerAvocadoTest, self).setUp() + + def checkLabel(self, key, value): + """ + check label of docker image, expect key value (could be read from config file) + + :param key: str + :param value: str + :return: bool + """ + if key in self.backend.containerInfo['Labels'] and ( + value in self.backend.containerInfo['Labels'][key]): + return True + return False + + diff --git a/moduleframework/avocado_testers/nspawn_avocado_test.py b/moduleframework/avocado_testers/nspawn_avocado_test.py new file mode 100644 index 0000000..91c395a --- /dev/null +++ b/moduleframework/avocado_testers/nspawn_avocado_test.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# +from moduleframework.module_framework import AvocadoTest + + +class NspawnAvocadoTest(AvocadoTest): + """ + Class for writing tests specific just for RPM module testing inside NSPAWN env + derived from AvocadoTest class. + + :avocado: disable + """ + + def setUp(self): + if self.moduleType != "nspawn": + self.skip("Nspawn specific test") + super(NspawnAvocadoTest, self).setUp() + + diff --git a/moduleframework/avocado_testers/rpm_avocado_test.py b/moduleframework/avocado_testers/rpm_avocado_test.py new file mode 100644 index 0000000..a484381 --- /dev/null +++ b/moduleframework/avocado_testers/rpm_avocado_test.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# + +from moduleframework.module_framework import AvocadoTest + + +class RpmAvocadoTest(AvocadoTest): + """ + Class for writing tests specific just for LOCAL (system) RPM testing + derived from AvocadoTest class. + + :avocado: disable + """ + + def setUp(self): + if self.moduleType != "rpm": + self.skip("Rpm specific test") + super(RpmAvocadoTest, self).setUp() + + diff --git a/moduleframework/common.py b/moduleframework/common.py index d9e815a..804439b 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -25,56 +25,15 @@ It provides some general functions """ -import sys import netifaces import socket import os -import linecache +import urllib +from avocado.utils import process -class ModuleFrameworkException(Exception): - def __init__(self, *args, **kwargs): - super(ModuleFrameworkException, self).__init__( - 'EXCEPTION MTF: ', *args, **kwargs) - exc_type, exc_obj, tb = sys.exc_info() - if tb is not None: - f = tb.tb_frame - lineno = tb.tb_lineno - filename = f.f_code.co_filename - linecache.checkcache(filename) - line = linecache.getline(filename, lineno, f.f_globals) - print "-----------\n| EXCEPTION IN: {} \n| LINE: {}, {} \n| ERROR: {}\n-----------".format(filename, lineno, line.strip(), exc_obj) - - -class NspawnExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(NspawnExc, self).__init__('TYPE nspawn', *args, **kwargs) - - -class RpmExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(RpmExc, self).__init__('TYPE rpm', *args, **kwargs) - - -class ContainerExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(ContainerExc, self).__init__('TYPE container', *args, **kwargs) - - -class ConfigExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(ConfigExc, self).__init__('TYPE config', *args, **kwargs) - - -class PDCExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(PDCExc, self).__init__('TYPE PDC', *args, **kwargs) - - -class KojiExc(ModuleFrameworkException): - def __init__(self, *args, **kwargs): - super(KojiExc, self).__init__('TYPE Koji', *args, **kwargs) - +from moduleframework.exceptions import * +from moduleframework.module_framework import get_profile defroutedev = netifaces.gateways().get('default').values( )[0][1] if netifaces.gateways().get('default') else "lo" @@ -231,3 +190,164 @@ def sanitize_cmd(cmd): if char in cmd: cmd = cmd.replace(char, '\\'.join(char)) return cmd + + +class CommonFunctions(object): + """ + Basic class doing configuration reading and allow do commands on host machine + """ + config = None + modulemdConf = None + + def __init__(self, *args, **kwargs): + self.config = None + self.modulemdConf = None + self.moduleName = None + self.source = None + self.arch = None + self.dependencylist = {} + self.moduledeps = None + # general use case is to have forwarded services to host (so thats why it is same) + self.ipaddr = trans_dict["HOSTIPADDR"] + trans_dict["GUESTARCH"] = self.getArch() + + def getArch(self): + """ + get system architecture string + + :return: str + """ + out = self.runHost(command='uname -m', verbose=False).stdout.strip() + return out + + def runHost(self, command="ls /", **kwargs): + """ + Run commands on host + + :param command: command to exectute + :param kwargs: (avocado process.run) params like: shell, ignore_status, verbose + :return: avocado.process.run + """ + try: + formattedcommand = command.format(**trans_dict) + except KeyError: + raise ModuleFrameworkException( + "Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ " + "or }}, possible values in trans_dict are:", + trans_dict) + return process.run("%s" % formattedcommand, **kwargs) + + def installTestDependencies(self, packages=None): + """ + Which packages install to host system to satisfy environment + + :param packages: List of packages, if not set, it will install rpms from config.yaml + :return: None + """ + if not packages: + typo = 'testdependecies' in self.config + if typo: + warnings.warn("'testdependecies' is a typo, please fix", + DeprecationWarning) + + # try section without typo first + packages = self.config.get('testdependencies', {}).get('rpms') + if packages: + if typo: + warnings.warn("preferring section without typo") + else: + # fall back to mistyped test dependency section + packages = self.config.get('testdependecies', {}).get('rpms') + + if packages: + self.runHost( + "{HOSTPACKAGER} install " + + " ".join(packages), + ignore_status=True, verbose=is_not_silent()) + + def loadconfig(self): + """ + Load configuration from config.yaml file (it is better to call this explicitly, than in + __init__ method for our purposes) + + :return: None + """ + try: + self.config = get_config() + self.moduleName = sanitize_text(self.config['name']) + self.source = self.config.get('source') if self.config.get( + 'source') else self.config['module']['rpm'].get('source') + except ValueError: + pass + + def getPackageList(self, profile=None): + """ + Return list of packages what has to be installed inside module + + :param profile: get list for intended profile instead of default method for searching + :return: list of packages (rpms) + """ + out = [] + if not profile: + if 'packages' in self.config: + packages_rpm = self.config['packages'].get('rpms') if self.config[ + 'packages'].get('rpms') else [] + packages_profiles = [] + for x in self.config['packages'].get('profiles') if self.config[ + 'packages'].get('profiles') else []: + packages_profiles = packages_profiles + \ + self.getModulemdYamlconfig()['data']['profiles'][x]['rpms'] + out += packages_rpm + packages_profiles + + elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data'][ + 'profiles'].get(get_profile()): + out += self.getModulemdYamlconfig()['data']['profiles'][get_profile()]['rpms'] + else: + # fallback solution when it is not known what to install + out.append("bash") + else: + out += self.getModulemdYamlconfig()['data']['profiles'][profile]['rpms'] + print_info("PCKGs to install inside module:", out) + return out + + def getModuleDependencies(self): + return self.dependencylist + + def getModulemdYamlconfig(self, urllink=None): + """ + Return moduleMD file yaml object. + It can be used also for loading another yaml file via url parameter + + :param urllink: load this url instead of default one defined in config, or redefined by vaiable CONFIG + :return: dict + """ + try: + if urllink: + ymlfile = urllib.urlopen(urllink) + cconfig = yaml.load(ymlfile) + link = cconfig + elif not get_if_module(): + trans_dict["GUESTPACKAGER"] = "yum -y" + link = {"data": {}} + else: + if self.config is None: + self.loadconfig() + if not self.modulemdConf: + modulemd = get_modulemd() + if modulemd: + ymlfile = urllib.urlopen(modulemd) + self.modulemdConf = yaml.load(ymlfile) + link = self.modulemdConf + return link + except IOError as e: + raise ConfigExc("Cannot load file") + + def getIPaddr(self): + """ + Return ip addr string of guest machine + In many cases it should be same as host machine and port should be forwarded to host + + :return: str + """ + return self.ipaddr + diff --git a/moduleframework/exceptions.py b/moduleframework/exceptions.py new file mode 100644 index 0000000..7b88010 --- /dev/null +++ b/moduleframework/exceptions.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Jan Scotka +# +from __future__ import print_function +import sys +import linecache + + +class ModuleFrameworkException(Exception): + def __init__(self, *args, **kwargs): + super(ModuleFrameworkException, self).__init__( + 'EXCEPTION MTF: ', *args, **kwargs) + exc_type, exc_obj, tb = sys.exc_info() + if tb is not None: + f = tb.tb_frame + lineno = tb.tb_lineno + filename = f.f_code.co_filename + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, f.f_globals) + print("-----------\n| EXCEPTION IN: {} \n| LINE: {}, {} \n| ERROR: {}\n-----------".format(filename, lineno, line.strip(), exc_obj)) + + +class NspawnExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(NspawnExc, self).__init__('TYPE nspawn', *args, **kwargs) + + +class RpmExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(RpmExc, self).__init__('TYPE rpm', *args, **kwargs) + + +class ContainerExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(ContainerExc, self).__init__('TYPE container', *args, **kwargs) + + +class ConfigExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(ConfigExc, self).__init__('TYPE config', *args, **kwargs) + + +class PDCExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(PDCExc, self).__init__('TYPE PDC', *args, **kwargs) + + +class KojiExc(ModuleFrameworkException): + def __init__(self, *args, **kwargs): + super(KojiExc, self).__init__('TYPE Koji', *args, **kwargs) + diff --git a/moduleframework/helpers/__init__.py b/moduleframework/helpers/__init__.py new file mode 100644 index 0000000..cbb80c5 --- /dev/null +++ b/moduleframework/helpers/__init__.py @@ -0,0 +1,22 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# diff --git a/moduleframework/helpers/container_helper.py b/moduleframework/helpers/container_helper.py new file mode 100644 index 0000000..c3e74fc --- /dev/null +++ b/moduleframework/helpers/container_helper.py @@ -0,0 +1,277 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# + +import re +import json + +from avocado.utils import service + +from moduleframework.module_framework import get_url +from moduleframework.common import * + + +class ContainerHelper(CommonFunctions): + """ + Basic Helper class for Docker container module type + + :avocado: disable + """ + + def __init__(self): + """ + set basic object variables + """ + super(ContainerHelper, self).__init__() + self.loadconfig() + self.info = self.config['module']['docker'] + self.tarbased = None + self.jmeno = None + self.docker_id = None + self.icontainer = get_url( + ) if get_url() else self.info['container'] + if ".tar" in self.icontainer: + self.jmeno = "testcontainer" + self.tarbased = True + if "docker=" in self.icontainer: + self.jmeno = self.icontainer[7:] + self.tarbased = False + elif "docker.io" in self.info['container']: + # Trusted source + self.tarbased = False + self.jmeno = self.icontainer + else: + # untrusted source + self.tarbased = False + self.jmeno = self.icontainer + + def getURL(self): + """ + It returns actual URL link string to container, It is same as URL + + :return: str + """ + return self.icontainer + + def getDockerInstanceName(self): + """ + Return docker instance name what will be used inside docker as docker image name + :return: str + """ + return self.jmeno + + def setUp(self): + """ + It is called by child class and it is same methof as Avocado/Unittest has. It prepares environment + for docker testing + * start docker if not + * pull docker image + * setup environment from config + * run and store identification + + :return: None + """ + self.installTestDependencies() + self.__callSetupFromConfig() + self.__prepare() + self.__prepareContainer() + self.__pullContainer() + + def tearDown(self): + """ + Cleanup environment and call also cleanup from config + + :return: None + """ + self.stop() + self.__callCleanupFromConfig() + + def __prepare(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if not os.path.isfile('/usr/bin/docker-current'): + self.runHost("{HOSTPACKAGER} install docker", verbose=is_not_silent()) + + def __prepareContainer(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.tarbased is False and self.jmeno == self.icontainer and "docker.io" not in self.info[ + 'container']: + registry = re.search("([^/]*)", self.icontainer).groups()[0] + if registry not in open('/etc/sysconfig/docker', 'rw').read(): + with open("/etc/sysconfig/docker", "a") as myfile: + myfile.write( + "INSECURE_REGISTRY='--insecure-registry $REGISTRY %s'" % + registry) + service_manager = service.ServiceManager() + service_manager.start('docker') + + def __pullContainer(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.tarbased: + self.runHost( + "docker import %s %s" % + (self.icontainer, self.jmeno), verbose=is_not_silent()) + elif "docker=" in self.icontainer: + pass + else: + self.runHost("docker pull %s" % self.jmeno, verbose=is_not_silent()) + + self.containerInfo = json.loads( + self.runHost( + "docker inspect %s" % + self.jmeno, verbose=is_not_silent()).stdout)[0]["Config"] + + def start(self, args="-it -d", command="/bin/bash"): + """ + start the docker container + + :param args: Do not use it directly (It is defined in config.yaml) + :param command: Do not use it directly (It is defined in config.yaml) + :return: None + """ + if not self.status(): + if 'start' in self.info and self.info['start']: + self.docker_id = self.runHost( + "%s -d %s" % + (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True, + verbose=is_not_silent()).stdout + else: + self.docker_id = self.runHost( + "docker run %s %s %s" % + (args, self.jmeno, command), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout + self.docker_id = self.docker_id.strip() + if self.getPackageList(): + a = self.run( + "%s install %s" % + (trans_dict["HOSTPACKAGER"], " ".join( + self.getPackageList())), + ignore_status=True, verbose=False) + b = self.run( + "%s install %s" % + (trans_dict["GUESTPACKAGER"], " ".join( + self.getPackageList())), + ignore_status=True, verbose=False) + if a.exit_status == 0: + print_info("Packages installed via {HOSTPACKAGER}", a.stdout) + elif b.exit_status == 0: + print_info("Packages installed via {GUESTPACKAGER}", b.stdout) + else: + print_info( + "Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", + self.getPackageList()) + if self.status() is False: + raise ContainerExc( + "Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % ( + self.jmeno, self.moduleName, self.docker_id)) + + def stop(self): + """ + Stop the docker container + + :return: None + """ + if self.status(): + try: + self.runHost("docker stop %s" % self.docker_id, verbose=is_not_silent()) + self.runHost("docker rm %s" % self.docker_id, verbose=is_not_silent()) + except Exception as e: + print_debug(e, "docker already removed") + pass + + def status(self): + """ + get status if container is running + + :return: bool + """ + if self.docker_id and self.docker_id[ + : 12] in self.runHost( + "docker ps", shell=True, verbose=is_not_silent()).stdout: + return True + else: + return False + + def run(self, command="ls /", **kwargs): + """ + Run command inside module, all params what allows avocado are passed inside shell,ignore_status, etc. + + :param command: str + :param kwargs: dict + :return: avocado.process.run + """ + self.start() + return self.runHost( + 'docker exec %s bash -c "%s"' % + (self.docker_id, sanitize_cmd(command)), + **kwargs) + + def copyTo(self, src, dest): + """ + Copy file to module + + :param src: str path to source file + :param dest: str path to file inside module + :return: None + """ + self.start() + self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest), verbose=is_not_silent()) + + def copyFrom(self, src, dest): + """ + Copy file from module + + :param src: str path of file inside module + :param dest: str path of destination file + :return: None + """ + self.start() + self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest), verbose=is_not_silent()) + + def __callSetupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("setup"): + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + + def __callCleanupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("cleanup"): + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + diff --git a/moduleframework/helpers/nspawn_helper.py b/moduleframework/helpers/nspawn_helper.py new file mode 100644 index 0000000..ed47442 --- /dev/null +++ b/moduleframework/helpers/nspawn_helper.py @@ -0,0 +1,395 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# + +import shutil +import re +import glob +import time + +from avocado.utils import process + +from timeoutlib import Retry +from moduleframework.common import * +from moduleframework.exceptions import * +from moduleframework.module_framework import get_if_do_cleanup +from moduleframework.helpers.rpm_helper import RpmHelper + + +class NspawnHelper(RpmHelper): + """ + Class for MODULE testing via NSPAWN created environment, it is type of virtualization, + something between chroot (MOCK) and full virtualization. For more info read: + https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html + + This class is derived from RPM HELPER, so that it uses same section in config file + """ + + def __init__(self): + """ + Set basic variables for NSPAWN environment, the most important is that it set + relative change root path + """ + super(NspawnHelper, self).__init__() + self.baseprefix = os.path.join(BASEPATHDIR, "chroot_") + self.__selinuxState = None + time.time() + actualtime = time.time() + if get_if_do_cleanup(): + self.jmeno = "%s_%r" % (self.moduleName, actualtime) + else: + self.jmeno = self.moduleName + self.chrootpath = os.path.abspath(self.baseprefix + self.jmeno) + print_info("name of CHROOT directory:", self.chrootpath) + trans_dict["ROOT"] = self.chrootpath + + def setUp(self): + """ + It is called by child class and it is same method as Avocado/Unittest has. It prepares environment + for systemd nspawn based testing + * installing dependencies from config + * setup environment from config + + :return: None + """ + + if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): + # TODO: workaround because systemd nspawn is now working well in F-25 + # (failing because of selinux) + self.__selinuxState = self.runHost( + "getenforce", ignore_status=True).stdout.strip() + self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent(), sudo=True) + self.setModuleDependencies() + self.setRepositoriesAndWhatToInstall() + self.installTestDependencies() + self.__prepareSetup() + self.__callSetupFromConfig() + self.__bootMachine() + + def __is_killed(self): + for foo in range(DEFAULTRETRYTIMEOUT): + time.sleep(1) + out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) + if out.exit_status != 0: + print_debug("NSPAWN machine %s stopped" % self.jmeno) + return True + raise NspawnExc("Unable to stop machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) + + def __is_booted(self): + for foo in range(DEFAULTRETRYTIMEOUT): + time.sleep(1) + out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) + if "systemd-logind" in out.stdout: + time.sleep(2) + print_debug("NSPAWN machine %s booted" % self.jmeno) + return True + raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) + + def __do_smart_start_cleanup(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + + if get_if_do_cleanup(): + # delete directory with same same (in case used option DO NOT CLEANUP) + if os.path.exists(self.chrootpath): + shutil.rmtree(self.chrootpath, ignore_errors=True) + # DELETE every chroot dir in case any exists + # Commented out, because it had side effect for multihost testing. Has to be improved + #dirstodelete = glob.glob(self.baseprefix + "*") + #if get_if_module() and dirstodelete: + # for dtd in dirstodelete: + # shutil.rmtree(dtd, ignore_errors=True) + os.mkdir(self.chrootpath) + + def __prepareSetup(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + self.__do_smart_start_cleanup() + if not os.path.exists(os.path.join(self.chrootpath, "usr")): + self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent(), sudo=True) + # workaround in case machined blocked by selinux, disabled for now + # self.runHost("sudo systemctl restart systemd-machined", verbose=is_not_silent(), sudo=True) + repos_to_use = "" + counter = 0 + for repo in self.repos: + counter = counter + 1 + repos_to_use += " --repofrompath %s%d,%s" % ( + self.moduleName, counter, repo) + try: + @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT * 60, delay=2 * 60, + error=NspawnExc("RETRY: Unable to install packages")) + def tmpfunc(): + self.runHost( + "%s install --nogpgcheck --setopt=install_weak_deps=False --installroot %s --allowerasing --disablerepo=* --enablerepo=%s* %s %s" % + (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, + self.whattoinstallrpm), verbose=is_not_silent()) + + tmpfunc() + except Exception as e: + raise NspawnExc( + "ERROR: Unable to install packages %s\n original exeption:\n%s\n" % + (self.whattoinstallrpm, str(e))) + # COPY yum repository inside NSPAW, to be able to do installations + insiderepopath = os.path.join(self.chrootpath, self.yumrepo[1:]) + try: + os.makedirs(os.path.dirname(insiderepopath)) + except: + pass + counter = 0 + f = open(insiderepopath, 'w') + for repo in self.repos: + counter = counter + 1 + add = """[%s%d] +name=%s%d +baseurl=%s +enabled=1 +gpgcheck=0 + +""" % (self.moduleName, counter, self.moduleName, counter, repo) + f.write(add) + f.close() + + # shutil.copy(self.yumrepo, insiderepopath) + # self.runHost("sed s/enabled=0/enabled=1/ -i %s" % insiderepopath, ignore_status=True) + for repo in self.repos: + if "file:///" in repo: + src = repo[7:] + srcto = os.path.join(self.chrootpath, src[1:]) + try: + os.makedirs(os.path.dirname(srcto)) + except Exception as e: + print_debug(e, "Unable to create DIR (already created)", srcto) + pass + try: + shutil.copytree(src, srcto) + except Exception as e: + print_debug(e, "Unable to copy files from:", src, "to:", srcto) + pass + pkipath = "/etc/pki/rpm-gpg" + pkipath_ch = os.path.join(self.chrootpath, pkipath[1:]) + try: + os.makedirs(pkipath_ch) + except BaseException: + pass + for filename in glob.glob(os.path.join(pkipath, '*')): + shutil.copy(filename, pkipath_ch) + print_info("repo prepared for microdnf:", insiderepopath, open(insiderepopath, 'r').read()) + + def __bootMachine(self): + + @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT, delay=21, + error=NspawnExc("RETRY: Unable to start nspawn machine")) + def tempfnc(): + print_debug("starting container via command:", + "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath)) + nspawncont = process.SubProcess( + "systemd-nspawn --machine=%s -bD %s" % + (self.jmeno, self.chrootpath), verbose=is_debug()) + nspawncont.start() + self.__is_booted() + + tempfnc() + print_info("machine: %s started" % self.jmeno) + + trans_dict["GUESTIPADDR"] = trans_dict["HOSTIPADDR"] + self.ipaddr = trans_dict["GUESTIPADDR"] + + def status(self, command="/bin/true"): + """ + Return status of module + + :param command: which command used for do that. it could be defined inside config + :return: bool + """ + try: + if 'status' in self.info and self.info['status']: + a = self.run(self.info['status'], shell=True, verbose=False, ignore_bg_processes=True) + else: + a = self.run("%s" % command, shell=True, verbose=False, ignore_bg_processes=True) + print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) + return True + except BaseException: + return False + + def start(self, command="/bin/true"): + """ + start the RPM based module (like systemctl start service) + + :param command: Do not use it directly (It is defined in config.yaml) + :return: None + """ + if 'start' in self.info and self.info['start']: + self.run(self.info['start'], shell=True, ignore_bg_processes=True) + else: + self.run("%s" % command, shell=True, ignore_bg_processes=True) + + def stop(self, command="/bin/true"): + """ + stop the RPM based module (like systemctl stop service) + + :param args: Do not use it directly (It is defined in config.yaml) + :param command: Do not use it directly (It is defined in config.yaml) + :return: None + """ + if 'stop' in self.info and self.info['stop']: + self.run(self.info['stop'], shell=True, ignore_bg_processes=True) + else: + self.run("%s" % command, shell=True, ignore_bg_processes=True) + + def run(self, command="ls /", **kwargs): + """ + Run command inside nspawn module type. It uses machinectl shell command. + It need few workarounds, that's why it the code seems so strange + + TODO: workaround because machinedctl is unable to behave like ssh. It is bug + systemd-run should be used, but in F-25 it does not contain --wait option + + :param command: str command to be executed + :param kwargs: dict parameters passed to avocado.process.run + :return: avocado.process.run + """ + lpath = "/var/tmp" + if not kwargs: + kwargs = {} + should_ignore = kwargs.get("ignore_status") + kwargs["ignore_status"] = True + + comout = self.runHost("""machinectl shell root@{machine} /bin/bash -c "({comm})>{pin}/stdout 2>{pin}/stderr; echo $?>{pin}/retcode; sleep 1" """.format( + machine=self.jmeno, + comm=sanitize_cmd(command), + pin=lpath), + **kwargs) + if comout.exit_status != 0: + raise NspawnExc("This command should not fail anyhow inside NSPAWN:", sanitize_cmd(command)) + try: + kwargs["verbose"] = is_not_silent() + b = self.runHost( + 'bash -c "cat {chroot}{pin}/stdout; cat {chroot}{pin}/stderr > /dev/stderr; exit `cat {chroot}{pin}/retcode`"'.format( + chroot=self.chrootpath, + pin=lpath), + **kwargs) + finally: + comout.stdout = b.stdout + comout.stderr = b.stderr + comout.exit_status = b.exit_status + removesworkaround = re.search('[^(]*\((.*)\)[^)]*', comout.command) + if removesworkaround: + comout.command = removesworkaround.group(1) + if comout.exit_status == 0 or should_ignore: + return comout + else: + raise process.CmdError(comout.command, comout) + + def selfcheck(self): + """ + Test if default command will pass, it is more important for nspawn, because it happens that + it does not returns anything + + :return: avocado.process.run + """ + return self.run().stdout + + def copyTo(self, src, dest): + """ + Copy file to module from host + + :param src: source file on host + :param dest: destination file on module + :return: None + """ + self.runHost( + " machinectl copy-to %s %s %s" % + (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) + + def copyFrom(self, src, dest): + """ + Copy file from module to host + + :param src: source file on module + :param dest: destination file on host + :return: None + """ + self.runHost( + " machinectl copy-from %s %s %s" % + (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) + + def tearDown(self): + """ + cleanup environment after test is finished and call cleanup section in config file + + :return: None + """ + try: + self.stop() + except Exception as stopexception: + print_info("STOP caused exception this is bad, but have to continue to terminate machine!!!", stopexception) + pass + + try: + self.runHost("machinectl poweroff %s" % self.jmeno, verbose=is_not_silent()) + self.__is_killed() + except Exception as poweroffex: + print_info("Unable to stop machine via poweroff, terminating", poweroffex) + try: + self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True) + self.__is_killed() + except Exception as poweroffexterm: + print_info("Unable to stop machine via terminate, STRANGE", poweroffexterm) + time.sleep(DEFAULTRETRYTIMEOUT) + pass + pass + + if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): + # TODO: workaround because systemd nspawn is now working well in F-25 + # (failing because of selinux) + self.runHost( + "setenforce %s" % + self.__selinuxState, + ignore_status=True, verbose=is_not_silent()) + if get_if_do_cleanup() and os.path.exists(self.chrootpath): + shutil.rmtree(self.chrootpath, ignore_errors=True) + self.__callCleanupFromConfig() + + def __callSetupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("setup"): + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + + def __callCleanupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("cleanup"): + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + diff --git a/moduleframework/helpers/rpm_helper.py b/moduleframework/helpers/rpm_helper.py new file mode 100644 index 0000000..8d59ac1 --- /dev/null +++ b/moduleframework/helpers/rpm_helper.py @@ -0,0 +1,278 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Petr Hracek +# + +import pdc_data + +from moduleframework.module_framework import get_url +from moduleframework.common import * +from moduleframework.exceptions import * + + +class RpmHelper(CommonFunctions): + """ + Class for testing "modules" on local machine (host) directly. It could be used for scheduling tests for + system packages + + :avocado: disable + """ + + def __init__(self): + """ + Set basic variables for RPM based testing, based on modules.rpm section of config.yaml + """ + super(RpmHelper, self).__init__() + self.loadconfig() + self.yumrepo = os.path.join( + "/etc", "yum.repos.d", "%s.repo" % + self.moduleName) + self.info = self.config['module']['rpm'] + self.repos = [] + self.whattoinstallrpm = "" + self.bootstrappackages = [] + + def setModuleDependencies(self): + temprepositories = {} + if self.getModulemdYamlconfig()["data"].get("dependencies") and self.getModulemdYamlconfig()["data"][ + "dependencies"].get("requires"): + temprepositories = self.getModulemdYamlconfig()["data"]["dependencies"]["requires"] + temprepositories_cycle = dict(temprepositories) + for x in temprepositories_cycle: + pdc = pdc_data.PDCParser() + pdc.setLatestPDC(x, temprepositories_cycle[x]) + temprepositories.update(pdc.generateDepModules()) + self.moduledeps = temprepositories + print_info("Detected module dependencies:", self.moduledeps) + + def getURL(self): + """ + Return semicolon separated string of repositories what will be used, could be simialr to URL param, + it contains also dependent repositories from PDC + + :return: str + """ + return ";".join(self.repos) + + def setUp(self): + """ + It is called by child class and it is same methof as Avocado/Unittest has. It prepares environment + for RPM based testing + * installing dependencies from config + * setup environment from config + + :return: None + """ + self.setModuleDependencies() + self.setRepositoriesAndWhatToInstall() + self.installTestDependencies() + self.__callSetupFromConfig() + self.__prepare() + self.__prepareSetup() + + def __addModuleDependency(self, url, name=None, stream="master"): + name = name if name else self.moduleName + if name in self.dependencylist: + self.dependencylist[name]['urls'].append(url) + else: + self.dependencylist[name] = {'urls':[url], 'stream':stream} + + + def setRepositoriesAndWhatToInstall(self, repos=None, whattooinstall=None): + """ + set repositories and packages what to install inside module + It can override base usage of this framework to general purpose testing + + :param repos: list of repositories + :param whattooinstall: list of packages to install inside + :return: None + """ + if repos is None: + repos = [] + alldrepos = [] + if repos: + self.repos = repos + map(self.__addModuleDependency, repos) + else: + if not self.repos: + for dep in self.moduledeps: + latesturl = pdc_data.get_repo_url(dep, self.moduledeps[dep]) + alldrepos.append(latesturl) + self.__addModuleDependency(url=latesturl, name = dep, stream = self.moduledeps[dep]) + if get_url(): + self.repos = [get_url()] + alldrepos + self.__addModuleDependency(get_url()) + elif self.info.get('repo'): + self.repos = [self.info.get('repo')] + alldrepos + self.__addModuleDependency(self.info.get('repo')) + elif self.info.get('repos'): + self.repos = self.info.get('repos') + map(self.__addModuleDependency,self.info.get('repos')) + else: + raise RpmExc("no RPM given in file or via URL") + if whattooinstall: + self.whattoinstallrpm = " ".join(set(whattooinstall)) + else: + if not self.whattoinstallrpm: + self.bootstrappackages = pdc_data.getBasePackageSet(modulesDict=self.moduledeps, + isModule=get_if_module(), isContainer=False) + self.whattoinstallrpm = " ".join(set(self.getPackageList() + self.bootstrappackages)) + + def tearDown(self): + """ + cleanup enviroment and call cleanup from config + + :return: None + """ + self.stop() + self.__callCleanupFromConfig() + + def __prepare(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + counter = 0 + f = open(self.yumrepo, 'w') + for repo in self.repos: + counter = counter + 1 + add = """[%s%d] +name=%s%d +baseurl=%s +enabled=1 +gpgcheck=0 + +""" % (self.moduleName, counter, self.moduleName, counter, repo) + f.write(add) + f.close() + + def __prepareSetup(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + + a = self.runHost( + "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % + (trans_dict["HOSTPACKAGER"], self.moduleName, self.whattoinstallrpm), ignore_status=True, + verbose=is_not_silent()) + b = self.runHost( + "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % + (trans_dict["HOSTPACKAGER"], self.moduleName), ignore_status=True, verbose=is_not_silent()) + + if a.exit_status != 0 and b.exit_status != 0: + raise RpmExc("ERROR: Unable to install packages %s" % self.whattoinstallrpm, + "repositories are: ", + self.runHost("cat %s" % self.yumrepo, verbose=is_not_silent()).stdout) + + self.ipaddr = trans_dict["GUESTIPADDR"] + + def status(self, command="/bin/true"): + """ + Return status of module + + :param command: which command used for do that. it could be defined inside config + :return: bool + """ + try: + if 'status' in self.info and self.info['status']: + a = self.runHost(self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + else: + a = self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) + return True + except BaseException: + return False + + def start(self, command="/bin/true"): + """ + start the RPM based module (like systemctl start service) + + :param command: Do not use it directly (It is defined in config.yaml) + :return: None + """ + if 'start' in self.info and self.info['start']: + self.runHost(self.info['start'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + else: + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + + def stop(self, command="/bin/true"): + """ + stop the RPM based module (like systemctl stop service) + + :param command: Do not use it directly (It is defined in config.yaml) + :return: None + """ + if 'stop' in self.info and self.info['stop']: + self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + else: + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + + def run(self, command="ls /", **kwargs): + """ + Run command inside module, for RPM based it is same as runHost + + :param command: str of command to execute + :param kwargs: dict from avocado.process.run + :return: avocado.process.run + """ + return self.runHost('bash -c "%s"' % + sanitize_cmd(command), **kwargs) + + def copyTo(self, src, dest): + """ + Copy file from one location (host) to another one to (module) + + :param src: str + :param dest: str + :return: None + """ + self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) + + def copyFrom(self, src, dest): + """ + Copy file from one location (module) to another one to (host) + + :param src: str + :param dest: str + :return: None + """ + self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) + + def __callSetupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("setup"): + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) + + def __callCleanupFromConfig(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + if self.info.get("cleanup"): + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 440306b..7c66e6f 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -26,22 +26,17 @@ main module provides helpers for various module types and AVOCADO(unittest) clas what you should use for your tests (inherited) """ -import re -import shutil import yaml -import json -import urllib -import glob from avocado import Test from avocado.core import exceptions -from avocado.utils import service -from avocado.utils import process -from compose_info import ComposeParser -import pdc_data -from common import * -from timeoutlib import Retry -import time -import warnings + +from moduleframework.compose_info import ComposeParser +from moduleframework.common import * +from moduleframework.exceptions import * +from moduleframework.helpers.container_helper import ContainerHelper +from moduleframework.helpers.nspawn_helper import NspawnHelper +from moduleframework.helpers.rpm_helper import RpmHelper + PROFILE = None @@ -59,1023 +54,6 @@ def skipTestIf(value, text="Test not intended for this module profile"): "DEPRECATED, don't use this skip, use self.cancel() inside test function, or self.skip() in setUp()") -class CommonFunctions(object): - """ - Basic class doing configuration reading and allow do commands on host machine - """ - config = None - modulemdConf = None - - def __init__(self, *args, **kwargs): - self.config = None - self.modulemdConf = None - self.moduleName = None - self.source = None - self.arch = None - self.dependencylist = {} - self.moduledeps = None - # general use case is to have forwarded services to host (so thats why it is same) - self.ipaddr = trans_dict["HOSTIPADDR"] - trans_dict["GUESTARCH"] = self.getArch() - - def getArch(self): - """ - get system architecture string - - :return: str - """ - out = self.runHost(command='uname -m', verbose=False).stdout.strip() - return out - - def runHost(self, command="ls /", **kwargs): - """ - Run commands on host - - :param command: command to exectute - :param kwargs: (avocado process.run) params like: shell, ignore_status, verbose - :return: avocado.process.run - """ - try: - formattedcommand = command.format(**trans_dict) - except KeyError: - raise ModuleFrameworkException( - "Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ " - "or }}, possible values in trans_dict are:", - trans_dict) - return process.run("%s" % formattedcommand, **kwargs) - - def installTestDependencies(self, packages=None): - """ - Which packages install to host system to satisfy environment - - :param packages: List of packages, if not set, it will install rpms from config.yaml - :return: None - """ - if not packages: - typo = 'testdependecies' in self.config - if typo: - warnings.warn("'testdependecies' is a typo, please fix", - DeprecationWarning) - - # try section without typo first - packages = self.config.get('testdependencies', {}).get('rpms') - if packages: - if typo: - warnings.warn("preferring section without typo") - else: - # fall back to mistyped test dependency section - packages = self.config.get('testdependecies', {}).get('rpms') - - if packages: - self.runHost( - "{HOSTPACKAGER} install " + - " ".join(packages), - ignore_status=True, verbose=is_not_silent()) - - def loadconfig(self): - """ - Load configuration from config.yaml file (it is better to call this explicitly, than in - __init__ method for our purposes) - - :return: None - """ - try: - self.config = get_config() - self.moduleName = sanitize_text(self.config['name']) - self.source = self.config.get('source') if self.config.get( - 'source') else self.config['module']['rpm'].get('source') - except ValueError: - pass - - def getPackageList(self, profile=None): - """ - Return list of packages what has to be installed inside module - - :param profile: get list for intended profile instead of default method for searching - :return: list of packages (rpms) - """ - out = [] - if not profile: - if 'packages' in self.config: - packages_rpm = self.config['packages'].get('rpms') if self.config[ - 'packages'].get('rpms') else [] - packages_profiles = [] - for x in self.config['packages'].get('profiles') if self.config[ - 'packages'].get('profiles') else []: - packages_profiles = packages_profiles + \ - self.getModulemdYamlconfig()['data']['profiles'][x]['rpms'] - out += packages_rpm + packages_profiles - - elif self.getModulemdYamlconfig()['data'].get('profiles') and self.getModulemdYamlconfig()['data'][ - 'profiles'].get(get_profile()): - out += self.getModulemdYamlconfig()['data']['profiles'][get_profile()]['rpms'] - else: - # fallback solution when it is not known what to install - out.append("bash") - else: - out += self.getModulemdYamlconfig()['data']['profiles'][profile]['rpms'] - print_info("PCKGs to install inside module:", out) - return out - - def getModuleDependencies(self): - return self.dependencylist - - def getModulemdYamlconfig(self, urllink=None): - """ - Return moduleMD file yaml object. - It can be used also for loading another yaml file via url parameter - - :param urllink: load this url instead of default one defined in config, or redefined by vaiable CONFIG - :return: dict - """ - try: - if urllink: - ymlfile = urllib.urlopen(urllink) - cconfig = yaml.load(ymlfile) - link = cconfig - elif not get_if_module(): - trans_dict["GUESTPACKAGER"] = "yum -y" - link = {"data": {}} - else: - if self.config is None: - self.loadconfig() - if not self.modulemdConf: - modulemd = get_modulemd() - if modulemd: - ymlfile = urllib.urlopen(modulemd) - self.modulemdConf = yaml.load(ymlfile) - link = self.modulemdConf - return link - except IOError as e: - raise ConfigExc("Cannot load file") - - def getIPaddr(self): - """ - Return ip addr string of guest machine - In many cases it should be same as host machine and port should be forwarded to host - - :return: str - """ - return self.ipaddr - - - - -class ContainerHelper(CommonFunctions): - """ - Basic Helper class for Docker container module type - - :avocado: disable - """ - - def __init__(self): - """ - set basic object variables - """ - super(ContainerHelper, self).__init__() - self.loadconfig() - self.info = self.config['module']['docker'] - self.tarbased = None - self.jmeno = None - self.docker_id = None - self.icontainer = get_url( - ) if get_url() else self.info['container'] - if ".tar" in self.icontainer: - self.jmeno = "testcontainer" - self.tarbased = True - if "docker=" in self.icontainer: - self.jmeno = self.icontainer[7:] - self.tarbased = False - elif "docker.io" in self.info['container']: - # Trusted source - self.tarbased = False - self.jmeno = self.icontainer - else: - # untrusted source - self.tarbased = False - self.jmeno = self.icontainer - - def getURL(self): - """ - It returns actual URL link string to container, It is same as URL - - :return: str - """ - return self.icontainer - - def getDockerInstanceName(self): - """ - Return docker instance name what will be used inside docker as docker image name - :return: str - """ - return self.jmeno - - def setUp(self): - """ - It is called by child class and it is same methof as Avocado/Unittest has. It prepares environment - for docker testing - * start docker if not - * pull docker image - * setup environment from config - * run and store identification - - :return: None - """ - self.installTestDependencies() - self.__callSetupFromConfig() - self.__prepare() - self.__prepareContainer() - self.__pullContainer() - - def tearDown(self): - """ - Cleanup environment and call also cleanup from config - - :return: None - """ - self.stop() - self.__callCleanupFromConfig() - - def __prepare(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if not os.path.isfile('/usr/bin/docker-current'): - self.runHost("{HOSTPACKAGER} install docker", verbose=is_not_silent()) - - def __prepareContainer(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.tarbased is False and self.jmeno == self.icontainer and "docker.io" not in self.info[ - 'container']: - registry = re.search("([^/]*)", self.icontainer).groups()[0] - if registry not in open('/etc/sysconfig/docker', 'rw').read(): - with open("/etc/sysconfig/docker", "a") as myfile: - myfile.write( - "INSECURE_REGISTRY='--insecure-registry $REGISTRY %s'" % - registry) - service_manager = service.ServiceManager() - service_manager.start('docker') - - def __pullContainer(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.tarbased: - self.runHost( - "docker import %s %s" % - (self.icontainer, self.jmeno), verbose=is_not_silent()) - elif "docker=" in self.icontainer: - pass - else: - self.runHost("docker pull %s" % self.jmeno, verbose=is_not_silent()) - - self.containerInfo = json.loads( - self.runHost( - "docker inspect %s" % - self.jmeno, verbose=is_not_silent()).stdout)[0]["Config"] - - def start(self, args="-it -d", command="/bin/bash"): - """ - start the docker container - - :param args: Do not use it directly (It is defined in config.yaml) - :param command: Do not use it directly (It is defined in config.yaml) - :return: None - """ - if not self.status(): - if 'start' in self.info and self.info['start']: - self.docker_id = self.runHost( - "%s -d %s" % - (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True, - verbose=is_not_silent()).stdout - else: - self.docker_id = self.runHost( - "docker run %s %s %s" % - (args, self.jmeno, command), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout - self.docker_id = self.docker_id.strip() - if self.getPackageList(): - a = self.run( - "%s install %s" % - (trans_dict["HOSTPACKAGER"], " ".join( - self.getPackageList())), - ignore_status=True, verbose=False) - b = self.run( - "%s install %s" % - (trans_dict["GUESTPACKAGER"], " ".join( - self.getPackageList())), - ignore_status=True, verbose=False) - if a.exit_status == 0: - print_info("Packages installed via {HOSTPACKAGER}", a.stdout) - elif b.exit_status == 0: - print_info("Packages installed via {GUESTPACKAGER}", b.stdout) - else: - print_info( - "Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", - self.getPackageList()) - if self.status() is False: - raise ContainerExc( - "Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % ( - self.jmeno, self.moduleName, self.docker_id)) - - def stop(self): - """ - Stop the docker container - - :return: None - """ - if self.status(): - try: - self.runHost("docker stop %s" % self.docker_id, verbose=is_not_silent()) - self.runHost("docker rm %s" % self.docker_id, verbose=is_not_silent()) - except Exception as e: - print_debug(e, "docker already removed") - pass - - def status(self): - """ - get status if container is running - - :return: bool - """ - if self.docker_id and self.docker_id[ - : 12] in self.runHost( - "docker ps", shell=True, verbose=is_not_silent()).stdout: - return True - else: - return False - - def run(self, command="ls /", **kwargs): - """ - Run command inside module, all params what allows avocado are passed inside shell,ignore_status, etc. - - :param command: str - :param kwargs: dict - :return: avocado.process.run - """ - self.start() - return self.runHost( - 'docker exec %s bash -c "%s"' % - (self.docker_id, sanitize_cmd(command)), - **kwargs) - - def copyTo(self, src, dest): - """ - Copy file to module - - :param src: str path to source file - :param dest: str path to file inside module - :return: None - """ - self.start() - self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest), verbose=is_not_silent()) - - def copyFrom(self, src, dest): - """ - Copy file from module - - :param src: str path of file inside module - :param dest: str path of destination file - :return: None - """ - self.start() - self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest), verbose=is_not_silent()) - - def __callSetupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - def __callCleanupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - -class RpmHelper(CommonFunctions): - """ - Class for testing "modules" on local machine (host) directly. It could be used for scheduling tests for - system packages - - :avocado: disable - """ - - def __init__(self): - """ - Set basic variables for RPM based testing, based on modules.rpm section of config.yaml - """ - super(RpmHelper, self).__init__() - self.loadconfig() - self.yumrepo = os.path.join( - "/etc", "yum.repos.d", "%s.repo" % - self.moduleName) - self.info = self.config['module']['rpm'] - self.repos = [] - self.whattoinstallrpm = "" - self.bootstrappackages = [] - - def setModuleDependencies(self): - temprepositories = {} - if self.getModulemdYamlconfig()["data"].get("dependencies") and self.getModulemdYamlconfig()["data"][ - "dependencies"].get("requires"): - temprepositories = self.getModulemdYamlconfig()["data"]["dependencies"]["requires"] - temprepositories_cycle = dict(temprepositories) - for x in temprepositories_cycle: - pdc = pdc_data.PDCParser() - pdc.setLatestPDC(x, temprepositories_cycle[x]) - temprepositories.update(pdc.generateDepModules()) - self.moduledeps = temprepositories - print_info("Detected module dependencies:", self.moduledeps) - - def getURL(self): - """ - Return semicolon separated string of repositories what will be used, could be simialr to URL param, - it contains also dependent repositories from PDC - - :return: str - """ - return ";".join(self.repos) - - def setUp(self): - """ - It is called by child class and it is same methof as Avocado/Unittest has. It prepares environment - for RPM based testing - * installing dependencies from config - * setup environment from config - - :return: None - """ - self.setModuleDependencies() - self.setRepositoriesAndWhatToInstall() - self.installTestDependencies() - self.__callSetupFromConfig() - self.__prepare() - self.__prepareSetup() - - def __addModuleDependency(self, url, name=None, stream="master"): - name = name if name else self.moduleName - if name in self.dependencylist: - self.dependencylist[name]['urls'].append(url) - else: - self.dependencylist[name] = {'urls':[url], 'stream':stream} - - - def setRepositoriesAndWhatToInstall(self, repos=None, whattooinstall=None): - """ - set repositories and packages what to install inside module - It can override base usage of this framework to general purpose testing - - :param repos: list of repositories - :param whattooinstall: list of packages to install inside - :return: None - """ - if repos is None: - repos = [] - alldrepos = [] - if repos: - self.repos = repos - map(self.__addModuleDependency, repos) - else: - if not self.repos: - for dep in self.moduledeps: - latesturl = pdc_data.get_repo_url(dep, self.moduledeps[dep]) - alldrepos.append(latesturl) - self.__addModuleDependency(url=latesturl, name = dep, stream = self.moduledeps[dep]) - if get_url(): - self.repos = [get_url()] + alldrepos - self.__addModuleDependency(get_url()) - elif self.info.get('repo'): - self.repos = [self.info.get('repo')] + alldrepos - self.__addModuleDependency(self.info.get('repo')) - elif self.info.get('repos'): - self.repos = self.info.get('repos') - map(self.__addModuleDependency,self.info.get('repos')) - else: - raise RpmExc("no RPM given in file or via URL") - if whattooinstall: - self.whattoinstallrpm = " ".join(set(whattooinstall)) - else: - if not self.whattoinstallrpm: - self.bootstrappackages = pdc_data.getBasePackageSet(modulesDict=self.moduledeps, - isModule=get_if_module(), isContainer=False) - self.whattoinstallrpm = " ".join(set(self.getPackageList() + self.bootstrappackages)) - - def tearDown(self): - """ - cleanup enviroment and call cleanup from config - - :return: None - """ - self.stop() - self.__callCleanupFromConfig() - - def __prepare(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - counter = 0 - f = open(self.yumrepo, 'w') - for repo in self.repos: - counter = counter + 1 - add = """[%s%d] -name=%s%d -baseurl=%s -enabled=1 -gpgcheck=0 - -""" % (self.moduleName, counter, self.moduleName, counter, repo) - f.write(add) - f.close() - - def __prepareSetup(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - - a = self.runHost( - "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % - (trans_dict["HOSTPACKAGER"], self.moduleName, self.whattoinstallrpm), ignore_status=True, - verbose=is_not_silent()) - b = self.runHost( - "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % - (trans_dict["HOSTPACKAGER"], self.moduleName), ignore_status=True, verbose=is_not_silent()) - - if a.exit_status != 0 and b.exit_status != 0: - raise RpmExc("ERROR: Unable to install packages %s" % self.whattoinstallrpm, - "repositories are: ", - self.runHost("cat %s" % self.yumrepo, verbose=is_not_silent()).stdout) - - self.ipaddr = trans_dict["GUESTIPADDR"] - - def status(self, command="/bin/true"): - """ - Return status of module - - :param command: which command used for do that. it could be defined inside config - :return: bool - """ - try: - if 'status' in self.info and self.info['status']: - a = self.runHost(self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - else: - a = self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) - return True - except BaseException: - return False - - def start(self, command="/bin/true"): - """ - start the RPM based module (like systemctl start service) - - :param command: Do not use it directly (It is defined in config.yaml) - :return: None - """ - if 'start' in self.info and self.info['start']: - self.runHost(self.info['start'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - def stop(self, command="/bin/true"): - """ - stop the RPM based module (like systemctl stop service) - - :param command: Do not use it directly (It is defined in config.yaml) - :return: None - """ - if 'stop' in self.info and self.info['stop']: - self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - def run(self, command="ls /", **kwargs): - """ - Run command inside module, for RPM based it is same as runHost - - :param command: str of command to execute - :param kwargs: dict from avocado.process.run - :return: avocado.process.run - """ - return self.runHost('bash -c "%s"' % - sanitize_cmd(command), **kwargs) - - def copyTo(self, src, dest): - """ - Copy file from one location (host) to another one to (module) - - :param src: str - :param dest: str - :return: None - """ - self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) - - def copyFrom(self, src, dest): - """ - Copy file from one location (module) to another one to (host) - - :param src: str - :param dest: str - :return: None - """ - self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) - - def __callSetupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - def __callCleanupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - -class NspawnHelper(RpmHelper): - """ - Class for MODULE testing via NSPAWN created environment, it is type of virtualization, - something between chroot (MOCK) and full virtualization. For more info read: - https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html - - This class is derived from RPM HELPER, so that it uses same section in config file - """ - - def __init__(self): - """ - Set basic variables for NSPAWN environment, the most important is that it set - relative change root path - """ - super(NspawnHelper, self).__init__() - self.baseprefix = os.path.join(BASEPATHDIR, "chroot_") - self.__selinuxState = None - time.time() - actualtime = time.time() - if get_if_do_cleanup(): - self.jmeno = "%s_%r" % (self.moduleName, actualtime) - else: - self.jmeno = self.moduleName - self.chrootpath = os.path.abspath(self.baseprefix + self.jmeno) - print_info("name of CHROOT directory:", self.chrootpath) - trans_dict["ROOT"] = self.chrootpath - - def setUp(self): - """ - It is called by child class and it is same method as Avocado/Unittest has. It prepares environment - for systemd nspawn based testing - * installing dependencies from config - * setup environment from config - - :return: None - """ - - if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): - # TODO: workaround because systemd nspawn is now working well in F-25 - # (failing because of selinux) - self.__selinuxState = self.runHost( - "getenforce", ignore_status=True).stdout.strip() - self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent(), sudo=True) - self.setModuleDependencies() - self.setRepositoriesAndWhatToInstall() - self.installTestDependencies() - self.__prepareSetup() - self.__callSetupFromConfig() - self.__bootMachine() - - def __is_killed(self): - for foo in range(DEFAULTRETRYTIMEOUT): - time.sleep(1) - out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) - if out.exit_status != 0: - print_debug("NSPAWN machine %s stopped" % self.jmeno) - return True - raise NspawnExc("Unable to stop machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) - - def __is_booted(self): - for foo in range(DEFAULTRETRYTIMEOUT): - time.sleep(1) - out = self.runHost("machinectl status %s" % self.jmeno, verbose=is_debug(), ignore_status=True) - if "systemd-logind" in out.stdout: - time.sleep(2) - print_debug("NSPAWN machine %s booted" % self.jmeno) - return True - raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) - - def __do_smart_start_cleanup(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - - if get_if_do_cleanup(): - # delete directory with same same (in case used option DO NOT CLEANUP) - if os.path.exists(self.chrootpath): - shutil.rmtree(self.chrootpath, ignore_errors=True) - # DELETE every chroot dir in case any exists - # Commented out, because it had side effect for multihost testing. Has to be improved - #dirstodelete = glob.glob(self.baseprefix + "*") - #if get_if_module() and dirstodelete: - # for dtd in dirstodelete: - # shutil.rmtree(dtd, ignore_errors=True) - os.mkdir(self.chrootpath) - - def __prepareSetup(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - self.__do_smart_start_cleanup() - if not os.path.exists(os.path.join(self.chrootpath, "usr")): - self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent(), sudo=True) - # workaround in case machined blocked by selinux, disabled for now - # self.runHost("sudo systemctl restart systemd-machined", verbose=is_not_silent(), sudo=True) - repos_to_use = "" - counter = 0 - for repo in self.repos: - counter = counter + 1 - repos_to_use += " --repofrompath %s%d,%s" % ( - self.moduleName, counter, repo) - try: - @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT * 60, delay=2 * 60, - error=NspawnExc("RETRY: Unable to install packages")) - def tmpfunc(): - self.runHost( - "%s install --nogpgcheck --setopt=install_weak_deps=False --installroot %s --allowerasing --disablerepo=* --enablerepo=%s* %s %s" % - (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, - self.whattoinstallrpm), verbose=is_not_silent()) - - tmpfunc() - except Exception as e: - raise NspawnExc( - "ERROR: Unable to install packages %s\n original exeption:\n%s\n" % - (self.whattoinstallrpm, str(e))) - # COPY yum repository inside NSPAW, to be able to do installations - insiderepopath = os.path.join(self.chrootpath, self.yumrepo[1:]) - try: - os.makedirs(os.path.dirname(insiderepopath)) - except: - pass - counter = 0 - f = open(insiderepopath, 'w') - for repo in self.repos: - counter = counter + 1 - add = """[%s%d] -name=%s%d -baseurl=%s -enabled=1 -gpgcheck=0 - -""" % (self.moduleName, counter, self.moduleName, counter, repo) - f.write(add) - f.close() - - # shutil.copy(self.yumrepo, insiderepopath) - # self.runHost("sed s/enabled=0/enabled=1/ -i %s" % insiderepopath, ignore_status=True) - for repo in self.repos: - if "file:///" in repo: - src = repo[7:] - srcto = os.path.join(self.chrootpath, src[1:]) - try: - os.makedirs(os.path.dirname(srcto)) - except Exception as e: - print_debug(e, "Unable to create DIR (already created)", srcto) - pass - try: - shutil.copytree(src, srcto) - except Exception as e: - print_debug(e, "Unable to copy files from:", src, "to:", srcto) - pass - pkipath = "/etc/pki/rpm-gpg" - pkipath_ch = os.path.join(self.chrootpath, pkipath[1:]) - try: - os.makedirs(pkipath_ch) - except BaseException: - pass - for filename in glob.glob(os.path.join(pkipath, '*')): - shutil.copy(filename, pkipath_ch) - print_info("repo prepared for microdnf:", insiderepopath, open(insiderepopath, 'r').read()) - - def __bootMachine(self): - - @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT, delay=21, - error=NspawnExc("RETRY: Unable to start nspawn machine")) - def tempfnc(): - print_debug("starting container via command:", - "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath)) - nspawncont = process.SubProcess( - "systemd-nspawn --machine=%s -bD %s" % - (self.jmeno, self.chrootpath), verbose=is_debug()) - nspawncont.start() - self.__is_booted() - - tempfnc() - print_info("machine: %s started" % self.jmeno) - - trans_dict["GUESTIPADDR"] = trans_dict["HOSTIPADDR"] - self.ipaddr = trans_dict["GUESTIPADDR"] - - def status(self, command="/bin/true"): - """ - Return status of module - - :param command: which command used for do that. it could be defined inside config - :return: bool - """ - try: - if 'status' in self.info and self.info['status']: - a = self.run(self.info['status'], shell=True, verbose=False, ignore_bg_processes=True) - else: - a = self.run("%s" % command, shell=True, verbose=False, ignore_bg_processes=True) - print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) - return True - except BaseException: - return False - - def start(self, command="/bin/true"): - """ - start the RPM based module (like systemctl start service) - - :param command: Do not use it directly (It is defined in config.yaml) - :return: None - """ - if 'start' in self.info and self.info['start']: - self.run(self.info['start'], shell=True, ignore_bg_processes=True) - else: - self.run("%s" % command, shell=True, ignore_bg_processes=True) - - def stop(self, command="/bin/true"): - """ - stop the RPM based module (like systemctl stop service) - - :param args: Do not use it directly (It is defined in config.yaml) - :param command: Do not use it directly (It is defined in config.yaml) - :return: None - """ - if 'stop' in self.info and self.info['stop']: - self.run(self.info['stop'], shell=True, ignore_bg_processes=True) - else: - self.run("%s" % command, shell=True, ignore_bg_processes=True) - - def run(self, command="ls /", **kwargs): - """ - Run command inside nspawn module type. It uses machinectl shell command. - It need few workarounds, that's why it the code seems so strange - - TODO: workaround because machinedctl is unable to behave like ssh. It is bug - systemd-run should be used, but in F-25 it does not contain --wait option - - :param command: str command to be executed - :param kwargs: dict parameters passed to avocado.process.run - :return: avocado.process.run - """ - lpath = "/var/tmp" - if not kwargs: - kwargs = {} - should_ignore = kwargs.get("ignore_status") - kwargs["ignore_status"] = True - - comout = self.runHost("""machinectl shell root@{machine} /bin/bash -c "({comm})>{pin}/stdout 2>{pin}/stderr; echo $?>{pin}/retcode; sleep 1" """.format( - machine=self.jmeno, - comm=sanitize_cmd(command), - pin=lpath), - **kwargs) - if comout.exit_status != 0: - raise NspawnExc("This command should not fail anyhow inside NSPAWN:", sanitize_cmd(command)) - try: - kwargs["verbose"] = is_not_silent() - b = self.runHost( - 'bash -c "cat {chroot}{pin}/stdout; cat {chroot}{pin}/stderr > /dev/stderr; exit `cat {chroot}{pin}/retcode`"'.format( - chroot=self.chrootpath, - pin=lpath), - **kwargs) - finally: - comout.stdout = b.stdout - comout.stderr = b.stderr - comout.exit_status = b.exit_status - removesworkaround = re.search('[^(]*\((.*)\)[^)]*', comout.command) - if removesworkaround: - comout.command = removesworkaround.group(1) - if comout.exit_status == 0 or should_ignore: - return comout - else: - raise process.CmdError(comout.command, comout) - - def selfcheck(self): - """ - Test if default command will pass, it is more important for nspawn, because it happens that - it does not returns anything - - :return: avocado.process.run - """ - return self.run().stdout - - def copyTo(self, src, dest): - """ - Copy file to module from host - - :param src: source file on host - :param dest: destination file on module - :return: None - """ - self.runHost( - " machinectl copy-to %s %s %s" % - (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) - - def copyFrom(self, src, dest): - """ - Copy file from module to host - - :param src: source file on module - :param dest: destination file on host - :return: None - """ - self.runHost( - " machinectl copy-from %s %s %s" % - (self.jmeno, src, dest), timeout=DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) - - def tearDown(self): - """ - cleanup environment after test is finished and call cleanup section in config file - - :return: None - """ - try: - self.stop() - except Exception as stopexception: - print_info("STOP caused exception this is bad, but have to continue to terminate machine!!!", stopexception) - pass - - try: - self.runHost("machinectl poweroff %s" % self.jmeno, verbose=is_not_silent()) - self.__is_killed() - except Exception as poweroffex: - print_info("Unable to stop machine via poweroff, terminating", poweroffex) - try: - self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True) - self.__is_killed() - except Exception as poweroffexterm: - print_info("Unable to stop machine via terminate, STRANGE", poweroffexterm) - time.sleep(DEFAULTRETRYTIMEOUT) - pass - pass - - if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): - # TODO: workaround because systemd nspawn is now working well in F-25 - # (failing because of selinux) - self.runHost( - "setenforce %s" % - self.__selinuxState, - ignore_status=True, verbose=is_not_silent()) - if get_if_do_cleanup() and os.path.exists(self.chrootpath): - shutil.rmtree(self.chrootpath, ignore_errors=True) - self.__callCleanupFromConfig() - - def __callSetupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - def __callCleanupFromConfig(self): - """ - Internal method, do not use it anyhow - - :return: None - """ - if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) - - # INTERFACE CLASS FOR GENERAL TESTS OF MODULES class AvocadoTest(Test): """ @@ -1280,62 +258,6 @@ class AvocadoTest(Test): return self.backend.getModuleDependencies() -# INTERFACE CLASSES FOR SPECIFIC MODULE TESTS -class ContainerAvocadoTest(AvocadoTest): - """ - Class for writing tests specific just for DOCKER - derived from AvocadoTest class. - - :avocado: disable - """ - - def setUp(self): - if self.moduleType != "docker": - self.skip("Docker specific test") - super(ContainerAvocadoTest, self).setUp() - - def checkLabel(self, key, value): - """ - check label of docker image, expect key value (could be read from config file) - - :param key: str - :param value: str - :return: bool - """ - if key in self.backend.containerInfo['Labels'] and ( - value in self.backend.containerInfo['Labels'][key]): - return True - return False - - -class RpmAvocadoTest(AvocadoTest): - """ - Class for writing tests specific just for LOCAL (system) RPM testing - derived from AvocadoTest class. - - :avocado: disable - """ - - def setUp(self): - if self.moduleType != "rpm": - self.skip("Rpm specific test") - super(RpmAvocadoTest, self).setUp() - - -class NspawnAvocadoTest(AvocadoTest): - """ - Class for writing tests specific just for RPM module testing inside NSPAWN env - derived from AvocadoTest class. - - :avocado: disable - """ - - def setUp(self): - if self.moduleType != "nspawn": - self.skip("Nspawn specific test") - super(NspawnAvocadoTest, self).setUp() - - def get_backend(): """ Return proper module type, set by config by default_module section, or defined via diff --git a/tools/modulelint.py b/tools/modulelint.py index 122f379..2a9e21c 100644 --- a/tools/modulelint.py +++ b/tools/modulelint.py @@ -20,10 +20,9 @@ # # Authors: Jan Scotka # - +from __future__ import print_function import os - from moduleframework import module_framework from moduleframework import dockerlinter @@ -106,11 +105,11 @@ class DockerLint(module_framework.ContainerAvocadoTest): """ llabels = self.getConfigModule().get('labels') if llabels is None or len(llabels) == 0: - print "No labels defined in config to check" + print("No labels defined in config to check") self.cancel() for key in self.getConfigModule()['labels']: aaa = self.checkLabel(key, self.getConfigModule()['labels'][key]) - print ">>>>>> ", aaa, key + print(">>>>>> ", aaa, key) self.assertTrue(aaa)