From 28af7a49a6aa9550ab732910b77750bb1dc42e1e Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Mar 24 2017 06:26:37 +0000 Subject: [PATCH 1/5] config improvement, firt commit to be able to disccuss about this --- diff --git a/docs/example-config-minimal.yaml b/docs/example-config-minimal.yaml new file mode 100644 index 0000000..ff63034 --- /dev/null +++ b/docs/example-config-minimal.yaml @@ -0,0 +1,14 @@ +document: modularity-testing +version: 1 +name: vim +source: https://github.com/container-images/memcached.git +modulemd-url: http://raw.githubusercontent.com/container-images/memcached/master/memcached.yaml +module: + docker: + labels: + description: "any description" + container: docker.io/phracek/memcached + rpm: + repos: + - https://phracek.fedorapeople.org/memcached-module-repo/ + diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 5c4669a..bacec99 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -36,8 +36,9 @@ class CommonFunctions(): def loadconfig(self): self.__modulemdConf = None self.config = get_correct_config() - self.packages = self.config['packages']['rpms'] self.moduleName = self.config['name'] + self.packages = self.config['packages']['rpms'] if self.config.has_key('packages') and self.config['packages'].has_key('rpms') and self.config['packages']['rpms'] else self.moduleName + self.source = self.config['source'] self.installTestDependencies() def getModulemdYamlconfig(self, urllink=None): From 5760875aadae39a8fe518a7e6829665b7636b3c4 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Mar 24 2017 07:14:57 +0000 Subject: [PATCH 2/5] added setup and cleanup part to config file (before starting module, after stopping) - is id done on host --- diff --git a/docs/example-config-minimal.yaml b/docs/example-config-minimal.yaml index ff63034..129fff5 100644 --- a/docs/example-config-minimal.yaml +++ b/docs/example-config-minimal.yaml @@ -9,6 +9,5 @@ module: description: "any description" container: docker.io/phracek/memcached rpm: - repos: - - https://phracek.fedorapeople.org/memcached-module-repo/ + repo: https://phracek.fedorapeople.org/memcached-module-repo/ diff --git a/docs/example-config.yaml b/docs/example-config.yaml index cfdee83..9b987e7 100644 --- a/docs/example-config.yaml +++ b/docs/example-config.yaml @@ -1,6 +1,7 @@ document: modularity-testing version: 1 name: memcached +source: https://github.com/container-images/memcached.git modulemd-url: http://raw.githubusercontent.com/container-images/memcached/master/memcached.yaml service: port: 11211 @@ -14,19 +15,22 @@ testdependecies: default_module: docker module: docker: + setup: echo Do magic with general config stored on host; + echo More magic + cleanup: echo Cleanup magic start: "docker run -it -e CACHE_SIZE=128 -p 11211:11211" labels: description: "memcached is a high-performance, distributed memory" io.k8s.description: "memcached is a high-performance, distributed memory" - source: https://github.com/container-images/memcached.git container: docker.io/phracek/memcached rpm: + setup: echo Do magic with general config stored on host; + echo More magic + cleanup: echo Cleanup magic start: systemctl start memcached stop: systemctl stop memcached status: systemctl status memcached - repos: - - http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/ - - https://phracek.fedorapeople.org/memcached-module-repo/ + repos: https://phracek.fedorapeople.org/memcached-module-repo/ test: processrunning: - 'ls /proc/*/exe -alh | grep memcached' diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index bacec99..9d5cf41 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -67,9 +67,11 @@ class ContainerHelper(CommonFunctions): self.__prepare() self.__prepareContainer() self.__pullContainer() + self.__callSetupFromConfig() def tearDown(self): self.stop() + self.__callCleanupFromConfig() def __prepare(self): if not os.path.isfile('/usr/bin/docker-current'): @@ -155,6 +157,13 @@ class ContainerHelper(CommonFunctions): self.start() self.runHost("docker cp %s:%s %s" % (self.docker_id, src, dest)) + def __callSetupFromConfig(self): + if self.info.get("setup"): + self.runHost(self.info.get("setup"), shell = True) + + def __callCleanupFromConfig(self): + if self.info.get("cleanup"): + self.runHost(self.info.get("cleanup"), shell = True) class RpmHelper(CommonFunctions): """ @@ -163,17 +172,19 @@ class RpmHelper(CommonFunctions): def setUp(self): self.loadconfig() - #self.installroot=os.path.join("/opt", self.moduleName) - # self.installroot="/" self.yumrepo = os.path.join( "/etc", "yum.repos.d", "%s.repo" % self.moduleName) self.info = self.config['module']['rpm'] + self.__baseruntimerepo = "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/" + self.__whattoinstallrpm = " ".join(self.getModulemdYamlconfig['data']['profiles']['rpms']) self.__prepare() self.__prepareSetup() + self.__callSetupFromConfig() def tearDown(self): self.stop() + self.__callCleanupFromConfig() def __prepare(self): # if not os.path.exists(self.installroot): @@ -182,7 +193,15 @@ class RpmHelper(CommonFunctions): if not os.path.isfile(self.yumrepo): counter = 0 f = open(self.yumrepo, 'w') - for repo in self.info['repos']: + if get_correct_url(): + repos = [get_correct_url(),self.__baseruntimerepo] + elif self.info.get('repo'): + repos = [self.info.get('repo'),self.__baseruntimerepo] + elif self.info.get('repos'): + repos = self.info.get('repos') + else: + raise ValueError ("no RPM given in file or via URL") + for repo in repos: counter = counter + 1 add = """[%s%d] name=%s%d @@ -195,10 +214,9 @@ gpgcheck=0 f.close() def __prepareSetup(self): - whattoinstall = " ".join(self.packages) + " rpm" utils.process.run( "dnf -y --disablerepo=* --enablerepo=%s* install %s" % - (self.moduleName, whattoinstall)) + (self.moduleName, self.__whattoinstallrpm)) def status(self, command="/bin/true"): if 'status' in self.info and self.info['status']: @@ -228,6 +246,15 @@ gpgcheck=0 def copyFrom(self, src, dest): self.runHost("cp -r %s %s" % (src, dest)) + def __callSetupFromConfig(self): + if self.info.get("setup"): + self.runHost(self.info.get("setup"), shell = True) + + def __callCleanupFromConfig(self): + if self.info.get("cleanup"): + self.runHost(self.info.get("cleanup"), shell = True) + + # INTERFACE CLASS FOR GENERAL TESTS OF MODULES class AvocadoTest(Test): From 4206a30e292d02007d45775b11820732ba755227 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Mar 24 2017 07:32:19 +0000 Subject: [PATCH 3/5] repaired mistakes caused that MODULE=rpm did not worked --- diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 9d5cf41..f3f98fb 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -177,7 +177,7 @@ class RpmHelper(CommonFunctions): self.moduleName) self.info = self.config['module']['rpm'] self.__baseruntimerepo = "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/" - self.__whattoinstallrpm = " ".join(self.getModulemdYamlconfig['data']['profiles']['rpms']) + self.__whattoinstallrpm = " ".join(self.getModulemdYamlconfig()['data']['profiles'][get_correct_profile()]['rpms']) self.__prepare() self.__prepareSetup() self.__callSetupFromConfig() From a202115ef7d1ec0a76fd2fe3968cbdbc420863ee Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Mar 24 2017 08:24:57 +0000 Subject: [PATCH 4/5] imporved docker handling, solved issues with missing labes in config --- diff --git a/docs/example-config-minimal.yaml b/docs/example-config-minimal.yaml index 129fff5..bfc20f6 100644 --- a/docs/example-config-minimal.yaml +++ b/docs/example-config-minimal.yaml @@ -1,12 +1,10 @@ document: modularity-testing version: 1 -name: vim +name: bash source: https://github.com/container-images/memcached.git modulemd-url: http://raw.githubusercontent.com/container-images/memcached/master/memcached.yaml module: docker: - labels: - description: "any description" container: docker.io/phracek/memcached rpm: repo: https://phracek.fedorapeople.org/memcached-module-repo/ diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile index e9df460..bd9a594 100644 --- a/examples/testing-module/Makefile +++ b/examples/testing-module/Makefile @@ -6,6 +6,15 @@ check-docker: check-rpm: MODULE=rpm $(CMD) +check-minimal-config-docker: + MODULE=docker CONFIG=./minimal.yaml $(CMD) + +check-minimal-config-rpm: + MODULE=rpm CONFIG=./minimal.yaml $(CMD) + +check-behave-docker: + cd ../memcached-behave; MODULE=docker behave + check: check-rpm all: check diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index f3f98fb..4a7fa8e 100644 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -37,8 +37,8 @@ class CommonFunctions(): self.__modulemdConf = None self.config = get_correct_config() self.moduleName = self.config['name'] - self.packages = self.config['packages']['rpms'] if self.config.has_key('packages') and self.config['packages'].has_key('rpms') and self.config['packages']['rpms'] else self.moduleName - self.source = self.config['source'] + self.packages = self.config['packages']['rpms'] if self.config.has_key('packages') and self.config['packages'].has_key('rpms') and self.config['packages']['rpms'] else [self.moduleName] + self.source = self.config.get('source') if self.config.get('source') else self.config['module']['rpm'].get('source') self.installTestDependencies() def getModulemdYamlconfig(self, urllink=None): @@ -117,7 +117,7 @@ class ContainerHelper(CommonFunctions): self.jmeno).stdout) def start(self, args="-it -d", command="/bin/bash"): - if not self.docker_id: + if not self.status(): if 'start' in self.info and self.info['start']: self.docker_id = utils.process.run( "%s -d %s" % @@ -129,19 +129,19 @@ class ContainerHelper(CommonFunctions): self.docker_id = self.docker_id.strip() def stop(self): - try: - utils.process.run("docker stop %s" % self.docker_id) - utils.process.run("docker rm %s" % self.docker_id) - except Exception as e: - print e - print "docker already removed" - pass + if self.status(): + try: + utils.process.run("docker stop %s" % self.docker_id) + utils.process.run("docker rm %s" % self.docker_id) + except Exception as e: + print e + print "docker already removed" + pass def status(self): - try: - self.run("true") + if self.docker_id and self.docker_id[:12] in self.runHost("docker ps", shell = True).stdout: return True - except Exception as e: + else: return False def run(self, command="ls /", **kwargs): diff --git a/moduleframework/modulelint.py b/moduleframework/modulelint.py index a71036c..b7371d5 100755 --- a/moduleframework/modulelint.py +++ b/moduleframework/modulelint.py @@ -26,12 +26,15 @@ class DockerLint(module_framework.ContainerAvocadoTest): self.assertIn(self.backend.jmeno, self.runHost("docker ps").stdout) def testLabels(self): + llabels = self.getConfigModule().get('labels') + module_framework.skipTestIf( llabels == None or len(llabels) == 0, "No labels defined in config to check") for key in self.getConfigModule()['labels']: aaa = self.checkLabel(key, self.getConfigModule()['labels'][key]) print ">>>>>> ", aaa, key self.assertTrue(aaa) + class ModuleLintSigning(module_framework.AvocadoTest): """ :avocado: enable From 6c93ff0ea0a3c7b4b3ca5868414a6adf8b0f039d Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Mar 24 2017 09:07:07 +0000 Subject: [PATCH 5/5] removed source section form minimal config, actually is not anyhow used --- diff --git a/docs/example-config-minimal.yaml b/docs/example-config-minimal.yaml index bfc20f6..84596b2 100644 --- a/docs/example-config-minimal.yaml +++ b/docs/example-config-minimal.yaml @@ -1,7 +1,6 @@ document: modularity-testing version: 1 name: bash -source: https://github.com/container-images/memcached.git modulemd-url: http://raw.githubusercontent.com/container-images/memcached/master/memcached.yaml module: docker: