From 5a9bb1ea487cfb1213874fb8d4d494fb42cbd688 Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 02 2017 12:48:47 +0000 Subject: [PATCH 1/6] Added Dockerfile linter Signed-off-by: Petr "Stone" Hracek --- diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 5ea8777..e4312d7 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -48,6 +48,7 @@ import warnings PROFILE = None + def skipTestIf(value, text="Test not intended for this module profile"): """ function what solves troubles that it is not possible to call SKIP inside code @@ -176,7 +177,7 @@ class CommonFunctions(object): return cconfig elif not get_if_module(): trans_dict["GUESTPACKAGER"] = "yum -y" - return {"data":{}} + return {"data": {}} else: if self.config is None: self.loadconfig() @@ -1228,7 +1229,6 @@ class NspawnAvocadoTest(AvocadoTest): super(NspawnAvocadoTest, self).setUp() - def get_correct_backend(): """ Return proper module type, set by config by default_module section, or defined via diff --git a/tools/modulelint.py b/tools/modulelint.py index 9c07f8d..0dd6501 100644 --- a/tools/modulelint.py +++ b/tools/modulelint.py @@ -21,8 +21,63 @@ # Authors: Jan Scotka # +import pprint +import os + from moduleframework import module_framework +from dockerfile_parse import DockerfileParser + +# Dockerfile path +DOCKERFILE = "Dockerfile" + +FROM = "FROM" +RUN = "RUN" + + +def _get_from(val): + if "baseruntime/baseruntime" in val: + return True + else: + return False + + +def _get_run(val): + if val.startswith("dnf") or " dnf " in val: + return False + else: + return True + +functions = {FROM: _get_from, + RUN: _get_run} + + +class DockerfileLinter(module_framework.ContainerAvocadoTest): + """ + :avocado: enable + """ + + def testDockerFromBaseruntime(self): + tmp_dir = os.path.join(os.getcwd(), "..") + Dockerfile = os.path.join(tmp_dir, DOCKERFILE) + if os.path.exists(Dockerfile): + dfp = DockerfileParser(path=tmp_dir) + for struct in dfp.structure: + key = struct["instruction"] + val = struct["value"] + if key == FROM: + self.assertTrue(functions[key](val)) + + def testDockerRunMicrodnf(self): + tmp_dir = os.path.join(os.getcwd(), "..") + Dockerfile = os.path.join(tmp_dir, DOCKERFILE) + if os.path.exists(Dockerfile): + dfp = DockerfileParser(path=tmp_dir) + for struct in dfp.structure: + key = struct["instruction"] + val = struct["value"] + if key == RUN and "dnf" in val: + self.assertTrue(functions[key](val)) class DockerLint(module_framework.ContainerAvocadoTest): From 492a380009308e5c69b2a1270780e8ee9b7c413e Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 06 2017 09:31:08 +0000 Subject: [PATCH 2/6] First draft of dockerlinter Signed-off-by: Petr "Stone" Hracek --- diff --git a/moduleframework/dockerlinter.py b/moduleframework/dockerlinter.py new file mode 100644 index 0000000..414d392 --- /dev/null +++ b/moduleframework/dockerlinter.py @@ -0,0 +1,164 @@ +from __future__ import absolute_import, print_function + +import os +import re +import ast + +from dockerfile_parse import DockerfileParser + +# Dockerfile path +DOCKERFILE = "Dockerfile" + +EXPOSE = "EXPOSE" +VOLUME = "VOLUME" +LABEL = "LABEL" +ENV = "ENV" +PORTS = "PORTS" +FROM = "FROM" +RUN = "RUN" + + +def get_string(value): + return ast.literal_eval(value) + + +class DockerLinter(object): + """ + Class checks a Dockerfile + It requires only directory with Dockerfile. + """ + + dockerfile = None + oc_template = None + dfp = {} + docker_dict = {} + + def __init__(self, dir_name=None): + self.dockerfile = os.path.join(dir_name, DOCKERFILE) + if not self._exist_docker_file(): + self.dfp = None + else: + self.dfp = DockerfileParser(path=dir_name) + self._get_structure_as_dict() + + def _exist_docker_file(self): + """ + Function checks if docker file exists + :return: True if exists + """ + if not os.path.exists(self.dockerfile): + print("Dockerfile has to exists in the %s directory." % self.dir) + return False + return True + + def _get_expose(self, value): + """Function returns exposes as field""" + return value.split() + + def _get_from(self, value): + """Function returns exposes as field""" + return value.split() + + def _get_run(self, value): + """Function returns exposes as field""" + return value.split() + + def _get_env(self, value): + """Function gets env as field""" + return value.split(" ") + + def _get_volume(self, value): + """Function evaluates a value and returns as string.""" + return get_string(value) + + def _get_label(self, val): + """ + Function returns label from Docker file + except INSTALL, UNINSTALL and RUN label used by atomic. + :param value: row from Dockerfile + :return: label_dict + """ + untracked_values = ['INSTALL', 'UNINSTALL', 'RUN'] + if [f for f in untracked_values if val.startswith(f)]: + return None + labels = re.sub('\s\s+', ';', val).split(';') + labels = [l.replace('"', '') for l in labels] + try: + label_dict = {l.split(' ')[0]: l.split(' ')[1] for l in labels} + except IndexError: + label_dict = {l.split('=')[0]: l.split('=')[1] for l in labels} + return label_dict + + def _get_structure_as_dict(self): + functions = {ENV: self._get_env, + EXPOSE: self._get_expose, + VOLUME: self._get_volume, + LABEL: self._get_label, + FROM: self._get_from, + RUN: self._get_run} + + for struct in self.dfp.structure: + key = struct["instruction"] + val = struct["value"] + if key == LABEL: + if key not in self.docker_dict: + self.docker_dict[key] = {} + value = functions[key](val) + if value is not None: + self.docker_dict[key].update(value) + else: + if key not in self.docker_dict: + self.docker_dict[key] = [] + try: + ret_val = functions[key](val) + for v in ret_val: + if v not in self.docker_dict[key]: + self.docker_dict[key].append(v) + except KeyError: + print("Dockerfile tag %s is not parsed by MTF" % key) + + def get_docker_env(self): + if ENV in self.docker_dict and self.docker_dict[ENV]: + return self.docker_dict[ENV] + + def get_docker_expose(self): + """ + Function return docker EXPOSE directives + :return: list of PORTS + """ + ports_list = [] + if EXPOSE in self.docker_dict and self.docker_dict[EXPOSE]: + for p in self.docker_dict[EXPOSE]: + ports_list.append(int(p)) + return ports_list + + def get_docker_labels(self): + """ + Function returns docker labels + :return: label dictionary + """ + if LABEL in self.docker_dict and self.docker_dict[LABEL]: + return self.docker_dict[LABEL] + return None + + def check_baseruntime(self): + """ + Function returns docker labels + :return: label dictionary + """ + if FROM in self.docker_dict: + return [x for x in self.docker_dict[FROM] if "baseruntime/baseruntime" in x] + + def check_microdnf(self): + """ + Function returns docker labels + :return: label dictionary + """ + if RUN in self.docker_dict: + for val in self.docker_dict[RUN]: + if val.startswith("dnf") or " dnf " in val: + return False + else: + return True + + diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index e4312d7..deed3d3 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -28,12 +28,10 @@ main module provides helpers for various module types and AVOCADO(unittest) clas what you should use for your tests (inherited) """ -import os import re import shutil import yaml import json -import time import urllib import glob from avocado import Test @@ -45,7 +43,6 @@ from common import * from timeoutlib import Retry import time import warnings - PROFILE = None diff --git a/tools/modulelint.py b/tools/modulelint.py index 0dd6501..3289442 100644 --- a/tools/modulelint.py +++ b/tools/modulelint.py @@ -21,63 +21,82 @@ # Authors: Jan Scotka # -import pprint import os from moduleframework import module_framework -from dockerfile_parse import DockerfileParser - -# Dockerfile path -DOCKERFILE = "Dockerfile" - -FROM = "FROM" -RUN = "RUN" - - -def _get_from(val): - if "baseruntime/baseruntime" in val: - return True - else: - return False - - -def _get_run(val): - if val.startswith("dnf") or " dnf " in val: - return False - else: - return True - -functions = {FROM: _get_from, - RUN: _get_run} +from moduleframework import dockerlinter class DockerfileLinter(module_framework.ContainerAvocadoTest): """ :avocado: enable + """ + dp = None + + def setUp(self): + # it is not intended just for docker, but just docker packages are + # actually properly signed + self.dp = dockerlinter.DockerLinter(os.path.join(os.getcwd(), "..")) + super(self.__class__, self).setUp() + def testDockerFromBaseruntime(self): - tmp_dir = os.path.join(os.getcwd(), "..") - Dockerfile = os.path.join(tmp_dir, DOCKERFILE) - if os.path.exists(Dockerfile): - dfp = DockerfileParser(path=tmp_dir) - for struct in dfp.structure: - key = struct["instruction"] - val = struct["value"] - if key == FROM: - self.assertTrue(functions[key](val)) + if self.dp is not None: + self.assertTrue(self.dp.check_baseruntime()) def testDockerRunMicrodnf(self): - tmp_dir = os.path.join(os.getcwd(), "..") - Dockerfile = os.path.join(tmp_dir, DOCKERFILE) - if os.path.exists(Dockerfile): - dfp = DockerfileParser(path=tmp_dir) - for struct in dfp.structure: - key = struct["instruction"] - val = struct["value"] - if key == RUN and "dnf" in val: - self.assertTrue(functions[key](val)) + if self.dp is not None: + self.assertTrue(self.dp.check_microdnf()) + + def testArchitectureInEnvAndLabelExists(self): + if self.dp is not None: + env_list = self.dp.get_docker_env() + self.assertTrue(x for x in env_list if "ARCH=" in x) + label_list = self.dp.get_docker_labels() + self.assertTrue("architecture" in label_list) + + def testNameInEnvAndLabelExists(self): + if self.dp is not None: + env_list = self.dp.get_docker_env() + self.assertTrue([x for x in env_list if "NAME=" in x]) + label_list = self.dp.get_docker_labels() + self.assertTrue("name" in label_list) + + def testReleaseLabelExists(self): + if self.dp is not None: + label_list = self.dp.get_docker_labels() + self.assertTrue("release" in label_list) + + def testVersionLabelExists(self): + if self.dp is not None: + label_list = self.dp.get_docker_labels() + self.assertTrue("version" in label_list) + + def testComRedHatComponentLabelExists(self): + if self.dp is not None: + label_list = self.dp.get_docker_labels() + self.assertTrue("com.redhat.component" in label_list) + + def testIok8sDescriptionExists(self): + if self.dp is not None: + label_list = self.dp.get_docker_labels() + self.assertTrue("io.k8s.description" in label_list) + + def testIoOpenshiftExposeServicesExists(self): + label_io_openshift = "io.openshift.expose-services" + if self.dp is not None: + exposes = self.dp.get_docker_expose() + label_list = self.dp.get_docker_labels() + self.assertTrue(label_list[label_io_openshift]) + for exp in exposes: + self.assertTrue("%s" % exp in label_list[label_io_openshift]) + + def testIoOpenShiftTagsExists(self): + if self.dp is not None: + label_list = self.dp.get_docker_labels() + self.assertTrue("io.openshift.tags" in label_list) class DockerLint(module_framework.ContainerAvocadoTest): From 97daac9a945b6e9661b6811c4fad56717ca47acc Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 06 2017 09:39:56 +0000 Subject: [PATCH 3/6] Add dependency into python2-dockerfile-parse Signed-off-by: Petr "Stone" Hracek --- diff --git a/distro/modularity-testing-framework.spec b/distro/modularity-testing-framework.spec index 6ecd027..2b99562 100644 --- a/distro/modularity-testing-framework.spec +++ b/distro/modularity-testing-framework.spec @@ -16,6 +16,7 @@ Requires: python2-avocado Requires: python2-avocado-plugins-output-html Requires: python-netifaces Requires: docker +Requires: python2-dockerfile-parse %description %{summary}. From cd36975ba70bf0ce4998233e6563e877cb283d7b Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 06 2017 11:22:14 +0000 Subject: [PATCH 4/6] Add dockerfile-parse into deps. Signed-off-by: Petr "Stone" Hracek --- diff --git a/setup.py b/setup.py index ada969b..83ff917 100755 --- a/setup.py +++ b/setup.py @@ -97,6 +97,7 @@ setup( install_requires=['avocado-framework', 'netifaces', 'behave', - 'PyYAML' + 'PyYAML', + 'dockerfile-parse' ] ) From 3f1701145476694a749a9678d0c88384bf025b21 Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 07 2017 07:31:40 +0000 Subject: [PATCH 5/6] Fixes according to comments. Signed-off-by: Petr "Stone" Hracek --- diff --git a/moduleframework/dockerlinter.py b/moduleframework/dockerlinter.py index 414d392..24b765c 100644 --- a/moduleframework/dockerlinter.py +++ b/moduleframework/dockerlinter.py @@ -22,7 +22,7 @@ def get_string(value): return ast.literal_eval(value) -class DockerLinter(object): +class DockerfileLinter(object): """ Class checks a Dockerfile It requires only directory with Dockerfile. @@ -51,16 +51,13 @@ class DockerLinter(object): return False return True - def _get_expose(self, value): - """Function returns exposes as field""" - return value.split() - - def _get_from(self, value): - """Function returns exposes as field""" - return value.split() - - def _get_run(self, value): - """Function returns exposes as field""" + def _get_general(self, value): + """ + Function returns exposes as field. + It is used for RUN, EXPOSE and FROM + :param value: + :return: + """ return value.split() def _get_env(self, value): @@ -91,11 +88,11 @@ class DockerLinter(object): def _get_structure_as_dict(self): functions = {ENV: self._get_env, - EXPOSE: self._get_expose, + EXPOSE: self._get_general, VOLUME: self._get_volume, LABEL: self._get_label, - FROM: self._get_from, - RUN: self._get_run} + FROM: self._get_general, + RUN: self._get_general} for struct in self.dfp.structure: key = struct["instruction"] @@ -121,6 +118,17 @@ class DockerLinter(object): if ENV in self.docker_dict and self.docker_dict[ENV]: return self.docker_dict[ENV] + def get_docker_specific_env(self, env_name=None): + """ + Function returns list of specific env_names or empty list + :param env_name: Specify env_name for check + :return: List of env or empty list + """ + if env_name is None: + return [] + env_list = self.get_docker_env() + return [env_name in env_list] + def get_docker_expose(self): """ Function return docker EXPOSE directives @@ -141,6 +149,17 @@ class DockerLinter(object): return self.docker_dict[LABEL] return None + def get_specific_label(self, label_name=None): + """ + Function returns list of specific label names or empty list + :param label_name: Specify label_name for check + :return: List of labels or empty list. + """ + if label_name is None: + return [] + label_list = self.get_docker_labels() + return [label_name in label_list] + def check_baseruntime(self): """ Function returns docker labels @@ -156,6 +175,8 @@ class DockerLinter(object): """ if RUN in self.docker_dict: for val in self.docker_dict[RUN]: + if val.startswith("yum") or " yum " in val: + return False if val.startswith("dnf") or " dnf " in val: return False else: diff --git a/tools/modulelint.py b/tools/modulelint.py index 3289442..f25868d 100644 --- a/tools/modulelint.py +++ b/tools/modulelint.py @@ -39,64 +39,48 @@ class DockerfileLinter(module_framework.ContainerAvocadoTest): def setUp(self): # it is not intended just for docker, but just docker packages are # actually properly signed - self.dp = dockerlinter.DockerLinter(os.path.join(os.getcwd(), "..")) super(self.__class__, self).setUp() + self.dp = dockerlinter.DockerLinter(os.path.join(os.getcwd(), "..")) + if self.dp is None: + self.skip() def testDockerFromBaseruntime(self): - if self.dp is not None: - self.assertTrue(self.dp.check_baseruntime()) + self.assertTrue(self.dp.check_baseruntime()) def testDockerRunMicrodnf(self): - if self.dp is not None: - self.assertTrue(self.dp.check_microdnf()) + self.assertTrue(self.dp.check_microdnf()) def testArchitectureInEnvAndLabelExists(self): - if self.dp is not None: - env_list = self.dp.get_docker_env() - self.assertTrue(x for x in env_list if "ARCH=" in x) - label_list = self.dp.get_docker_labels() - self.assertTrue("architecture" in label_list) + self.assertTrue(self.dp.get_docker_specific_env("ARCH=")) + self.assertTrue(self.dp.get_specific_label("architecture")) def testNameInEnvAndLabelExists(self): - if self.dp is not None: - env_list = self.dp.get_docker_env() - self.assertTrue([x for x in env_list if "NAME=" in x]) - label_list = self.dp.get_docker_labels() - self.assertTrue("name" in label_list) + self.assertTrue(self.dp.get_docker_specific_env("NAME=")) + self.assertTrue(self.dp.get_specific_label("name")) def testReleaseLabelExists(self): - if self.dp is not None: - label_list = self.dp.get_docker_labels() - self.assertTrue("release" in label_list) + self.assertTrue(self.dp.get_specific_label("release")) def testVersionLabelExists(self): - if self.dp is not None: - label_list = self.dp.get_docker_labels() - self.assertTrue("version" in label_list) + self.assertTrue(self.dp.get_specific_label("version")) def testComRedHatComponentLabelExists(self): - if self.dp is not None: - label_list = self.dp.get_docker_labels() - self.assertTrue("com.redhat.component" in label_list) + self.assertTrue(self.dp.get_specific_label("com.redhat.component")) def testIok8sDescriptionExists(self): - if self.dp is not None: - label_list = self.dp.get_docker_labels() - self.assertTrue("io.k8s.description" in label_list) + self.assertTrue(self.dp.get_specific_label("io.k8s.description")) def testIoOpenshiftExposeServicesExists(self): label_io_openshift = "io.openshift.expose-services" - if self.dp is not None: - exposes = self.dp.get_docker_expose() - label_list = self.dp.get_docker_labels() - self.assertTrue(label_list[label_io_openshift]) - for exp in exposes: - self.assertTrue("%s" % exp in label_list[label_io_openshift]) + exposes = self.dp.get_docker_expose() + label_list = self.dp.get_docker_labels() + self.assertTrue(label_list[label_io_openshift]) + for exp in exposes: + self.assertTrue("%s" % exp in label_list[label_io_openshift]) def testIoOpenShiftTagsExists(self): - if self.dp is not None: - label_list = self.dp.get_docker_labels() - self.assertTrue("io.openshift.tags" in label_list) + label_list = self.dp.get_docker_labels() + self.assertTrue("io.openshift.tags" in label_list) class DockerLint(module_framework.ContainerAvocadoTest): @@ -109,10 +93,18 @@ class DockerLint(module_framework.ContainerAvocadoTest): self.assertTrue("bin" in self.run("ls /").stdout) def testContainerIsRunning(self): + """ + Function tests whether container is running + :return: + """ self.start() self.assertIn(self.backend.jmeno, self.runHost("docker ps").stdout) def testLabels(self): + """ + Function tests whether labels are set in modulemd YAML file properly. + :return: + """ llabels = self.getConfigModule().get('labels') if llabels is None or len(llabels) == 0: print "No labels defined in config to check" From 280da180e97b9e2af6d7b2bc96717d846ace31b1 Mon Sep 17 00:00:00 2001 From: Petr "Stone" Hracek Date: Jun 07 2017 07:53:46 +0000 Subject: [PATCH 6/6] Merge branch 'master' into dockerlinter --- diff --git a/.tito/packages/modularity-testing-framework b/.tito/packages/modularity-testing-framework index da4ad2c..d05d092 100644 --- a/.tito/packages/modularity-testing-framework +++ b/.tito/packages/modularity-testing-framework @@ -1 +1 @@ -0.4.43-1 distro/ +0.4.52-1 distro/ diff --git a/Vagrantfile b/Vagrantfile index b297b49..d2c0929 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -24,11 +24,11 @@ Vagrant.configure(2) do |config| config.vm.box = "fedora/25-cloud-base" - config.vm.synced_folder ".", "/vagrant" + config.vm.synced_folder ".", "/home/vagrant" config.vm.network "private_network", ip: "192.168.50.10" config.vm.network "forwarded_port", guest: 80, host: 8888 config.vm.hostname = "moduletesting" - config.vm.post_up_message = "Results: http://localhost:8888/job-results" + config.vm.post_up_message = "Results: http://localhost:8888/avocado/job-results/latest/html/results.html" config.vm.provider "libvirt" do |libvirt| libvirt.memory = 1024 @@ -42,9 +42,10 @@ Vagrant.configure(2) do |config| config.vm.provision "shell", inline: <<-SHELL set -x - dnf install -y python-pip make docker httpd git python2-avocado python2-avocado-plugins-output-html - cd /vagrant - make all + dnf install -y make docker httpd git python2-avocado python2-avocado-plugins-output-html python-netifaces + cd /home/vagrant + make install + make check cp -r /root/avocado /var/www/html/ chmod -R a+x /var/www/html/ restorecon -r /var/www/html/ diff --git a/distro/modularity-testing-framework.spec b/distro/modularity-testing-framework.spec index 2b99562..c3caf74 100644 --- a/distro/modularity-testing-framework.spec +++ b/distro/modularity-testing-framework.spec @@ -1,7 +1,7 @@ %global framework_name moduleframework Name: modularity-testing-framework -Version: 0.4.43 +Version: 0.4.52 Release: 1%{?dist} Summary: Framework for writing tests for modules and containers @@ -45,6 +45,33 @@ chmod a+x %{buildroot}%{python_sitelib}/%{framework_name}/{module_framework,gene %{_datadir}/moduleframework/ %changelog +* Fri Jun 02 2017 Jan Scotka 0.4.52-1 +- + +* Thu Jun 01 2017 Jan Scotka 0.4.51-1 +- + +* Thu Jun 01 2017 Jan Scotka 0.4.50-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.49-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.48-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.47-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.46-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.45-1 +- + +* Wed May 31 2017 Jan Scotka 0.4.44-1 +- + * Tue May 30 2017 Jan Scotka 0.4.43-1 - diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile index b5d25e0..2310184 100644 --- a/examples/testing-module/Makefile +++ b/examples/testing-module/Makefile @@ -1,21 +1,21 @@ CMD=python -m avocado run --filter-by-tags=-WIP TESTS=$(shell ls *.py *.sh modulelint/*.py) export MTF_REMOTE_REPOS=yes -export MTF_DO_NOT_CLEANUP=yes export DEBUG=yes check-docker: MODULE=docker $(CMD) $(TESTS) @true -check-rpm: - MODULE=nspawn $(CMD) $(TESTS) - @true - check-minimal-config-docker: MODULE=docker CONFIG=./minimal.yaml $(CMD) $(TESTS) @true + +check-rpm: + MODULE=nspawn $(CMD) $(TESTS) + @true + check-minimal-config-rpm: MODULE=nspawn CONFIG=./minimal.yaml $(CMD) $(TESTS) @true @@ -25,19 +25,23 @@ check-behave-docker: @true check-multihost-testing: - cd ../multios_testing; MTF_DO_NOT_CLEANUP= MTF_DISABLE_MODULE=yes $(CMD) *.py + cd ../multios_testing; MTF_DISABLE_MODULE=yes $(CMD) *.py @true check-run-them-pdc-baseruntime: - MTF_REMOTE_REPOS= MTF_DO_NOT_CLEANUP= ../../tools/run-them.sh base-runtime base-runtime-f26-20170504201340 pdc + ../../tools/run-them.sh base-runtime base-runtime-f26-20170504201340 pdc @true check-run-them-pdc-testmodule: - MTF_REMOTE_REPOS= ../../tools/run-them.sh testmodule testmodule-master-20170413142055 pdc + ../../tools/run-them.sh testmodule testmodule-master-20170413142055 pdc @true check-run-them-fedmsg-testmodule: - MTF_REMOTE_REPOS= ../../tools/run-them.sh testmodule ../../tools/example_message_module.yaml fedmsg + ../../tools/run-them.sh testmodule ../../tools/example_message_module.yaml fedmsg + @true + +check-exceptions: + MODULEMDURL=XXX MODULE=nspawn $(CMD) --show-job-log simpleTest.py | grep 'raise ConfigExc' &>/dev/null @true diff --git a/examples/testing-module/copyTest.py b/examples/testing-module/copyTest.py index 5c49095..39e8c15 100644 --- a/examples/testing-module/copyTest.py +++ b/examples/testing-module/copyTest.py @@ -31,6 +31,10 @@ class CheckCopyFiles(module_framework.AvocadoTest): def testCopyThereAndBack(self): self.start() + #cleanup of all files, because there is bug in nspawn copying of files causing that it hang in case of existing file (F-25) + self.runHost("rm a b", ignore_status=True) + self.run("rm /a.test", ignore_status=True) + self.runHost("echo x > a", shell=True) self.copyTo("a", "/a.test") self.assertIn("x", self.run("cat /a.test").stdout) diff --git a/moduleframework/common.py b/moduleframework/common.py index adc1d7d..38f800b 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -32,6 +32,37 @@ import netifaces import socket import os + +class ModuleFrameworkException(Exception): + def __init__(self,*args,**kwargs): + super(ModuleFrameworkException, self).__init__(*args,**kwargs) + print_info('EXCEPTION nspawn', *args) + +class NspawnExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(NspawnExc, self).__init__('EXCEPTION nspawn', *args,**kwargs) + +class RpmExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(RpmExc, self).__init__('EXCEPTION rpm dnf yum', *args,**kwargs) + +class ContainerExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(ContainerExc, self).__init__('EXCEPTION container', *args,**kwargs) + +class ConfigExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(ConfigExc, self).__init__('EXCEPTION config', *args,**kwargs) + +class PDCExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(PDCExc, self).__init__('EXCEPTION PDC', *args,**kwargs) + +class KojiExc(ModuleFrameworkException): + def __init__(self,*args,**kwargs): + super(KojiExc, self).__init__('EXCEPTION Koji', *args,**kwargs) + + defroutedev = netifaces.gateways().get('default').values( )[0][1] if netifaces.gateways().get('default') else "lo" hostipaddr = netifaces.ifaddresses(defroutedev)[2][0]['addr'] @@ -46,6 +77,7 @@ if os.path.exists('/usr/bin/dnf'): # translation table for config.yaml files syntax is {VARIABLE} in config file trans_dict = {"HOSTIPADDR": hostipaddr, + "GUESTIPADDR": hostipaddr, "DEFROUTE": defroutedev, "HOSTNAME": hostname, "ROOT": "/", @@ -70,6 +102,9 @@ DEFAULTNSPAWNTIMEOUT = 10 def is_debug(): return bool(os.environ.get("DEBUG")) +def is_not_silent(): + return not is_debug() + def print_info(*args): """ Print data to selected output in case you are not in testing class, there is self.log @@ -83,7 +118,7 @@ def print_info(*args): try: out = arg.format(**trans_dict) except KeyError: - raise BaseException( + raise ModuleFrameworkException( "String is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ or }}, possible values in trans_dict are:", trans_dict) print >> sys.stderr, out diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index deed3d3..11a5a19 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -36,6 +36,7 @@ import urllib import glob from avocado import Test from avocado import utils +from avocado.core import exceptions from avocado.utils import service from compose_info import ComposeParser import pdc_data @@ -56,7 +57,7 @@ def skipTestIf(value, text="Test not intended for this module profile"): :return: None """ if value: - raise BaseException("DEPRECATED, don't use this skip, use self.cancel() inside test function, or self.skip() in setUp()") + raise ModuleFrameworkException("DEPRECATED, don't use this skip, use self.cancel() inside test function, or self.skip() in setUp()") class CommonFunctions(object): @@ -85,7 +86,7 @@ class CommonFunctions(object): try: formattedcommand = command.format(**trans_dict) except KeyError: - raise BaseException("Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ or }}, possible values in trans_dict are:", trans_dict) + raise ModuleFrameworkException("Command is formatted by using trans_dict, if you want to use brackets { } in your code please use {{ or }}, possible values in trans_dict are:", trans_dict) return utils.process.run("%s" % formattedcommand, **kwargs) def installTestDependencies(self, packages=None): @@ -114,7 +115,7 @@ class CommonFunctions(object): self.runHost( "{HOSTPACKAGER} install " + " ".join(packages), - ignore_status=True) + ignore_status=True, verbose=is_not_silent()) def loadconfig(self): """ @@ -168,22 +169,26 @@ class CommonFunctions(object): :param urllink: load this url instead of default one defined in config, or redefined by vaiable CONFIG :return: dict """ - if urllink: - ymlfile = urllib.urlopen(urllink) - cconfig = yaml.load(ymlfile) - return cconfig - elif not get_if_module(): - trans_dict["GUESTPACKAGER"] = "yum -y" - return {"data": {}} - else: - if self.config is None: - self.loadconfig() - if not self.modulemdConf: - modulemd = get_correct_modulemd() - if modulemd: - ymlfile = urllib.urlopen(modulemd) - self.modulemdConf = yaml.load(ymlfile) - return self.modulemdConf + try: + if urllink: + ymlfile = urllib.urlopen(urllink) + cconfig = yaml.load(ymlfile) + link = cconfig + elif not get_if_module(): + trans_dict["GUESTPACKAGER"] = "yum -y" + link = {"data": {}} + else: + if self.config is None: + self.loadconfig() + if not self.modulemdConf: + modulemd = get_correct_modulemd() + if modulemd: + ymlfile = urllib.urlopen(modulemd) + self.modulemdConf = yaml.load(ymlfile) + link = self.modulemdConf + return link + except IOError as e: + raise ConfigExc("Cannot load file") def getIPaddr(self): """ @@ -277,7 +282,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if not os.path.isfile('/usr/bin/docker-current'): - self.runHost("{HOSTPACKAGER} install docker") + self.runHost("{HOSTPACKAGER} install docker",verbose=is_not_silent()) def __prepareContainer(self): """ @@ -305,16 +310,16 @@ class ContainerHelper(CommonFunctions): if self.tarbased: self.runHost( "docker import %s %s" % - (self.icontainer, self.jmeno)) + (self.icontainer, self.jmeno), verbose=is_not_silent()) elif "docker=" in self.icontainer: pass else: - self.runHost("docker pull %s" % self.jmeno) + self.runHost("docker pull %s" % self.jmeno, verbose=is_not_silent()) self.containerInfo = json.loads( self.runHost( "docker inspect %s" % - self.jmeno).stdout)[0]["Config"] + self.jmeno, verbose=is_not_silent()).stdout)[0]["Config"] def start(self, args="-it -d", command="/bin/bash"): """ @@ -328,11 +333,11 @@ class ContainerHelper(CommonFunctions): if 'start' in self.info and self.info['start']: self.docker_id = self.runHost( "%s -d %s" % - (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True).stdout + (self.info['start'], self.jmeno), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout else: self.docker_id = self.runHost( "docker run %s %s %s" % - (args, self.jmeno, command), shell=True, ignore_bg_processes=True).stdout + (args, self.jmeno, command), shell=True, ignore_bg_processes=True, verbose=is_not_silent()).stdout self.docker_id = self.docker_id.strip() if self.getPackageList(): a = self.run( @@ -352,7 +357,7 @@ class ContainerHelper(CommonFunctions): else: print_info("Nothing installed (nor via {HOSTPACKAGER} nor {GUESTPACKAGER}), but package list is not empty", self.getPackageList()) if self.status() is False: - raise BaseException("Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % (self.jmeno, self.moduleName, self.docker_id)) + raise ContainerExc("Container %s (for module %s) is not running, probably DEAD immediately after start (ID: %s)" % (self.jmeno, self.moduleName, self.docker_id)) def stop(self): """ @@ -362,8 +367,8 @@ class ContainerHelper(CommonFunctions): """ if self.status(): try: - self.runHost("docker stop %s" % self.docker_id) - self.runHost("docker rm %s" % self.docker_id) + self.runHost("docker stop %s" % self.docker_id, verbose=is_not_silent()) + self.runHost("docker rm %s" % self.docker_id, verbose=is_not_silent()) except Exception as e: print_debug(e, "docker already removed") pass @@ -376,7 +381,7 @@ class ContainerHelper(CommonFunctions): """ if self.docker_id and self.docker_id[ : 12] in self.runHost( - "docker ps", shell=True).stdout: + "docker ps", shell=True, verbose=is_not_silent()).stdout: return True else: return False @@ -404,7 +409,7 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest)) + self.runHost("docker cp %s %s:%s" % (src, self.docker_id, dest), verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -415,7 +420,7 @@ class ContainerHelper(CommonFunctions): :return: None """ self.start() - self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest)) + self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest), verbose=is_not_silent()) def __callSetupFromConfig(self): """ @@ -424,7 +429,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -433,7 +438,7 @@ class ContainerHelper(CommonFunctions): :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) class RpmHelper(CommonFunctions): @@ -519,7 +524,7 @@ class RpmHelper(CommonFunctions): elif self.info.get('repos'): self.repos = self.info.get('repos') else: - raise ValueError("no RPM given in file or via URL") + raise RpmExc("no RPM given in file or via URL") if whattooinstall: self.whattoinstallrpm = " ".join(set(whattooinstall)) else: @@ -562,21 +567,20 @@ gpgcheck=0 :return: None """ - try: - self.runHost( - "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % - (trans_dict["HOSTPACKAGER"],self.moduleName, self.whattoinstallrpm)) - self.runHost( - "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % - (trans_dict["HOSTPACKAGER"], self.moduleName), ignore_status=True) - except Exception as e: - raise Exception( - "ERROR: Unable to install packages %s from repositories \n%s\n original exeption:\n%s\n" % - (self.whattoinstallrpm, - self.runHost( - "cat %s" % - self.yumrepo).stdout, - e)) + + a = self.runHost( + "%s --disablerepo=* --enablerepo=%s* --allowerasing install %s" % + (trans_dict["HOSTPACKAGER"],self.moduleName, self.whattoinstallrpm), ignore_status=True, verbose=is_not_silent()) + b =self.runHost( + "%s --disablerepo=* --enablerepo=%s* --allowerasing distro-sync" % + (trans_dict["HOSTPACKAGER"], self.moduleName), ignore_status=True, verbose=is_not_silent()) + + if a.exit_status != 0 and b.exit_status != 0: + raise RpmExc("ERROR: Unable to install packages %s" % self.whattoinstallrpm, + "repositories are: ", + self.runHost("cat %s" % self.yumrepo, verbose=is_not_silent()).stdout) + + self.ipaddr = trans_dict["GUESTIPADDR"] def status(self, command="/bin/true"): """ @@ -587,9 +591,9 @@ gpgcheck=0 """ try: if 'status' in self.info and self.info['status']: - a = self.runHost(self.info['status'], shell=True, verbose=False, ignore_bg_processes=True) + a = self.runHost(self.info['status'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - a = self.runHost("%s" % command, shell=True, verbose=False, ignore_bg_processes=True) + a = self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) print_debug("command:",a.command ,"stdout:",a.stdout, "stderr:", a.stderr) return True except BaseException: @@ -604,9 +608,9 @@ gpgcheck=0 :return: None """ if 'start' in self.info and self.info['start']: - self.runHost(self.info['start'], shell=True, ignore_bg_processes=True) + self.runHost(self.info['start'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True) + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def stop(self, command="/bin/true"): """ @@ -617,9 +621,9 @@ gpgcheck=0 :return: None """ if 'stop' in self.info and self.info['stop']: - self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True) + self.runHost(self.info['stop'], shell=True, ignore_bg_processes=True, verbose=is_not_silent()) else: - self.runHost("%s" % command, shell=True, ignore_bg_processes=True) + self.runHost("%s" % command, shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def run(self, command="ls /", **kwargs): """ @@ -640,7 +644,7 @@ gpgcheck=0 :param dest: str :return: None """ - self.runHost("cp -r %s %s" % (src, dest)) + self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -650,7 +654,7 @@ gpgcheck=0 :param dest: str :return: None """ - self.runHost("cp -r %s %s" % (src, dest)) + self.runHost("cp -r %s %s" % (src, dest), verbose=is_not_silent()) def __callSetupFromConfig(self): """ @@ -659,7 +663,7 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -668,7 +672,7 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) class NspawnHelper(RpmHelper): @@ -713,12 +717,13 @@ class NspawnHelper(RpmHelper): # (failing because of selinux) self.__selinuxState = self.runHost( "getenforce", ignore_status=True).stdout.strip() - self.runHost("setenforce Permissive", ignore_status=True) + self.runHost("setenforce Permissive", ignore_status=True, verbose=is_not_silent()) self.setModuleDependencies() self.setRepositoriesAndWhatToInstall() self.installTestDependencies() self.__prepareSetup() self.__callSetupFromConfig() + self.__bootMachine() def __is_killed(self): for foo in range(DEFAULTRETRYTIMEOUT): @@ -727,7 +732,7 @@ class NspawnHelper(RpmHelper): if out.exit_status != 0: print_debug("NSPAWN machine %s stopped" % self.jmeno) return True - raise BaseException("Unable to stop machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to stop machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) def __is_booted(self): for foo in range(DEFAULTRETRYTIMEOUT): @@ -737,7 +742,7 @@ class NspawnHelper(RpmHelper): time.sleep(2) print_debug("NSPAWN machine %s booted" % self.jmeno) return True - raise BaseException("Unable to start machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) + raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno,DEFAULTRETRYTIMEOUT)) def __prepareSetup(self): """ @@ -749,12 +754,12 @@ class NspawnHelper(RpmHelper): shutil.rmtree(self.chrootpath, ignore_errors=True) os.mkdir(self.chrootpath) try: - self.runHost("machinectl terminate %s" % self.jmeno) + self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug()) self.__is_killed() except BaseException: pass if not os.path.exists(os.path.join(self.chrootpath, "usr")): - self.runHost("{HOSTPACKAGER} install systemd-container") + self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent()) repos_to_use = "" counter = 0 for repo in self.repos: @@ -762,19 +767,21 @@ class NspawnHelper(RpmHelper): repos_to_use += " --repofrompath %s%d,%s" % ( self.moduleName, counter, repo) try: - self.runHost( - "%s --nogpgcheck install --installroot %s --allowerasing --disablerepo=* --enablerepo=%s* %s %s" % - (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, self.whattoinstallrpm)) + @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT*60, delay=2*60, error=NspawnExc("RETRY: Unable to install packages")) + def tmpfunc(): + self.runHost( + "%s install --nogpgcheck --setopt=install_weak_deps=False --installroot %s --allowerasing --disablerepo=* --enablerepo=%s* %s %s" % + (trans_dict["HOSTPACKAGER"], self.chrootpath, self.moduleName, repos_to_use, self.whattoinstallrpm), verbose=is_not_silent()) + tmpfunc() except Exception as e: - raise Exception( + raise NspawnExc( "ERROR: Unable to install packages %s\n original exeption:\n%s\n" % (self.whattoinstallrpm, str(e))) # COPY yum repository inside NSPAW, to be able to do installations insiderepopath = os.path.join(self.chrootpath, self.yumrepo[1:]) try: os.makedirs(os.path.dirname(insiderepopath)) - except Exception as e: - print_debug(e) + except: pass counter = 0 f = open(insiderepopath, 'w') @@ -816,17 +823,25 @@ gpgcheck=0 shutil.copy(filename, pkipath_ch) print_info("repo prepared for microdnf:", insiderepopath, open(insiderepopath, 'r').read()) - @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT, delay=21, error=Exception("Timeout: Unable to start nspawn machine")) + def __bootMachine(self): + + @Retry(attempts=DEFAULTRETRYCOUNT, timeout=DEFAULTRETRYTIMEOUT, delay=21, + error=NspawnExc("RETRY: Unable to start nspawn machine")) def tempfnc(): - print_debug("starting container via command:", "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath)) + print_debug("starting container via command:", + "systemd-nspawn --machine=%s -bD %s" % (self.jmeno, self.chrootpath)) nspawncont = utils.process.SubProcess( "systemd-nspawn --machine=%s -bD %s" % - (self.jmeno, self.chrootpath)) + (self.jmeno, self.chrootpath), verbose=is_debug()) nspawncont.start() self.__is_booted() + tempfnc() print_info("machine: %s started" % self.jmeno) + trans_dict["GUESTIPADDR"] = trans_dict["HOSTIPADDR"] + self.ipaddr = trans_dict["GUESTIPADDR"] + def status(self, command="/bin/true"): """ Return status of module @@ -894,7 +909,7 @@ gpgcheck=0 try: if not kwargs: kwargs = {} - kwargs["verbose"]=False + kwargs["verbose"]=is_not_silent() should_ignore=kwargs.get("ignore_status") kwargs["ignore_status"]=True b = self.runHost( @@ -912,7 +927,7 @@ gpgcheck=0 if comout.exit_status == 0 or should_ignore: return comout else: - utils.process.CmdError(comout.command, comout) + raise utils.process.CmdError(comout.command, comout) def selfcheck(self): """ @@ -933,7 +948,7 @@ gpgcheck=0 """ self.runHost( " machinectl copy-to %s %s %s" % - (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True) + (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) def copyFrom(self, src, dest): """ @@ -945,7 +960,7 @@ gpgcheck=0 """ self.runHost( " machinectl copy-from %s %s %s" % - (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True) + (self.jmeno, src, dest), timeout = DEFAULTPROCESSTIMEOUT, ignore_bg_processes=True, verbose=is_not_silent()) def tearDown(self): """ @@ -954,7 +969,7 @@ gpgcheck=0 :return: None """ self.stop() - self.runHost("machinectl poweroff %s" % self.jmeno) + self.runHost("machinectl poweroff %s" % self.jmeno, verbose=is_not_silent()) # self.nspawncont.stop() self.__is_killed() if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): @@ -963,7 +978,7 @@ gpgcheck=0 self.runHost( "setenforce %s" % self.__selinuxState, - ignore_status=True) + ignore_status=True, verbose=is_not_silent()) if get_if_do_cleanup() and os.path.exists(self.chrootpath): shutil.rmtree(self.chrootpath, ignore_errors=True) self.__callCleanupFromConfig() @@ -976,7 +991,7 @@ gpgcheck=0 :return: None """ if self.info.get("setup"): - self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("setup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) def __callCleanupFromConfig(self): """ @@ -985,7 +1000,7 @@ gpgcheck=0 :return: None """ if self.info.get("cleanup"): - self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True) + self.runHost(self.info.get("cleanup"), shell=True, ignore_bg_processes=True, verbose=is_not_silent()) # INTERFACE CLASS FOR GENERAL TESTS OF MODULES @@ -1003,16 +1018,19 @@ class AvocadoTest(Test): :avocado: disable """ def __init__(self,*args, **kwargs): - @Retry(attempts=1,timeout=55) - def tmpfunc(): - super(AvocadoTest,self).__init__(*args, **kwargs) - (self.backend, self.moduleType) = get_correct_backend() - self.moduleProfile = get_correct_profile() - print_info( - "Module Type: %s; Profile: %s" % - (self.moduleType, self.moduleProfile)) - tmpfunc() + super(AvocadoTest,self).__init__(*args, **kwargs) + + (self.backend, self.moduleType) = get_correct_backend() + self.moduleProfile = get_correct_profile() + print_info( + "Module Type: %s; Profile: %s" % + (self.moduleType, self.moduleProfile)) + def cancel(self, *args, **kwargs): + try: + super(AvocadoTest, self).cancel(*args, **kwargs) + except AttributeError: + raise exceptions.TestDecoratorSkip(*args, **kwargs) def setUp(self): """ @@ -1137,7 +1155,7 @@ class AvocadoTest(Test): :return: str """ self.start() - allpackages = self.run(r'rpm -qa --qf="%{{name}}\n"').stdout.split('\n') + allpackages = self.run(r'rpm -qa --qf="%{{name}}\n"', verbose=is_not_silent()).stdout.split('\n') return allpackages def copyTo(self, *args, **kwargs): @@ -1246,7 +1264,7 @@ def get_correct_backend(): elif amodule == 'nspawn': return NspawnHelper(), amodule else: - raise ValueError("Unsupported MODULE={0}".format(amodule)) + raise ModuleFrameworkException("Unsupported MODULE={0}".format(amodule), "supproted are: docker, rpm, nspawn") def get_correct_profile(): @@ -1286,13 +1304,13 @@ def get_correct_config(): if not cfgfile: cfgfile = "config.yaml" if not os.path.exists(cfgfile): - raise ValueError( + raise ConfigExc( "Config file (%s) does not exist or is inaccesible (you can also redefine own by CONFIG=path/to/configfile.yaml env variable)" % cfgfile) with open(cfgfile, 'r') as ymlfile: xcfg = yaml.load(ymlfile.read()) if xcfg['document'] != 'modularity-testing': - raise ValueError( + raise ConfigExc( "Bad Config file, not yaml or does not contain proper document type" % cfgfile) return xcfg diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index 7651db0..13fb55d 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -80,7 +80,7 @@ class PDCParser(): Class for parsing PDC data via some setters line setFullVersion, setViaFedMsg, setLatestPDC """ - @Retry(attempts=DEFAULTRETRYCOUNT*5, timeout=DEFAULTRETRYTIMEOUT, delay=20) + @Retry(attempts=DEFAULTRETRYCOUNT*5, timeout=DEFAULTRETRYTIMEOUT, delay=20, error=PDCExc("RETRY: Unable to get data from PDC")) def __getDataFromPdc(self): """ Internal method, do not use it @@ -89,12 +89,12 @@ class PDCParser(): """ PDC = "%s/?variant_name=%s&variant_version=%s&variant_release=%s&active=True" % ( PDCURL, self.name, self.stream, self.version) - print_debug("attemt to contact PDC with:", PDC) + print_info("Attemt to contact PDC (may take longer time) with query:", PDC) out=json.load(urllib.urlopen(PDC))["results"] if out: self.pdcdata = out[-1] else: - raise BaseException("Unable to get data from PDC URL: %s" % PDC) + raise PDCExc("Unable to get data from PDC URL: %s" % PDC) def setFullVersion(self, nvr): """ @@ -142,8 +142,9 @@ class PDCParser(): :return: str """ - rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % ( - self.pdcdata["koji_tag"] + "-build", ARCH) + #rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % ( + # self.pdcdata["koji_tag"] + "-build", ARCH) + rpmrepo = "https://kojipkgs.stg.fedoraproject.org/compose/branched/jkaluza/latest-Fedora-Modular-26/compose/Server/%s/os/" % ARCH return rpmrepo def generateGitHash(self): @@ -220,7 +221,7 @@ class PDCParser(): if len(pkgbouid) > 4: print_info("DOWNLOADING: %s" % foo) - @Retry(attempts=DEFAULTRETRYCOUNT*10, timeout=DEFAULTRETRYTIMEOUT*60, delay=DEFAULTRETRYTIMEOUT, error=Exception("Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT*10))) + @Retry(attempts=DEFAULTRETRYCOUNT*10, timeout=DEFAULTRETRYTIMEOUT*60, delay=DEFAULTRETRYTIMEOUT, error=KojiExc("RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT*10))) def tmpfunc(): a = utils.process.run( "cd %s; koji download-build %s -a %s -a noarch" % @@ -229,7 +230,7 @@ class PDCParser(): if "packages available for" in a.stdout.strip(): print_info('UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: - raise BaseException('UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) + raise KojiExc('UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() utils.process.run( "cd %s; createrepo -v %s" % diff --git a/tools/modulelint/check_compose.py b/tools/modulelint/check_compose.py index 23cf6b5..1c09ef1 100644 --- a/tools/modulelint/check_compose.py +++ b/tools/modulelint/check_compose.py @@ -42,12 +42,15 @@ class ComposeTest(module_framework.NspawnAvocadoTest): """ self.log.info("Checking availability of component and installation and remove them") for profile in self.getModulemdYamlconfig()["data"].get("profiles"): - actualpackagelist = " ".join( - set(self.getModulemdYamlconfig()["data"]["profiles"].get(profile)) - -set(self.backend.bootstrappackages) - ) + actualpackagelist = set(self.getModulemdYamlconfig()["data"]["profiles"][profile]["rpms"]) - set(self.backend.bootstrappackages) packager = common.trans_dict["GUESTPACKAGER"] if actualpackagelist: - self.run("%s install %s" % (packager, actualpackagelist)) - self.run("rpm -q %s" % actualpackagelist) - self.run("%s remove %s" % (packager, actualpackagelist)) + checkpackage = self.run("rpm -q --qf='%{{name}}\\n' " + " ".join(actualpackagelist), ignore_status=True).stdout.split() + installed = [x for x in checkpackage if "not installed" not in x] + self.log.info("Already installed packages:", installed) + + actualpackages = " ".join(list(set(actualpackagelist)-set(installed))) + if len(actualpackages)>2: + self.run("%s install %s" % (packager, actualpackages)) + self.run("rpm -q %s" % actualpackagelist) + self.run("%s remove %s" % (packager, actualpackages))