From 2777480f15e7768f60c33b6f38e099516857a9f1 Mon Sep 17 00:00:00 2001 From: clime Date: Jan 15 2021 19:41:51 +0000 Subject: [PATCH 1/4] add base of spec preprocessing functionality * enabled by setting preprocess_spec to True in the config file * uses preproc and rpkg-macros to do the job * when preprocessing is enabled, the generated files will be put into an auto-generated directory under /tmp/ (e.g. /tmp/fedpkg for the fedpkg tool) * this enables preprocessing for all rpm-based commands: srpm, local, prep, install, compile, lint, verify-files, clog mockbuild, koji scratch build, copr build * also a new subcommand `spec` is introduced just to render the spec file without further building srpm etc. * some introduced functions are currently unused (e.g. utils.changelog_entry or utils.edit). The functions are intended to be used in a future commit that extends the `tag` subcommand to automatically pregenerate tag messages. * preproc and rpkg-macros packages are newly required, also python-munch Signed-off-by: clime --- diff --git a/README.rst b/README.rst index 8219fd8..cae5ea6 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,8 @@ packages are required to be installed as well. * ``rpmlint``: check SPEC. * ``copr-cli``: for building package in `Fedora Copr`_. * ``module-build-service``: for building modules. +* ``preproc``: for spec file preprocessing +* ``rpkg-macros``: for spec file preprocessing .. _`Fedora Copr`: https://copr.fedorainfracloud.org/ diff --git a/etc/bash_completion.d/rpkg.bash b/etc/bash_completion.d/rpkg.bash index 18cdf47..97013bf 100644 --- a/etc/bash_completion.d/rpkg.bash +++ b/etc/bash_completion.d/rpkg.bash @@ -35,7 +35,7 @@ _rpkg() local options="--help -v -q" local options_value="--dist --release --user --path" local commands="build chain-build ci clean clog clone co container-build container-build-config commit compile copr-build diff flatpak-build \ - gimmespec giturl help gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push retire scratch-build sources \ + gimmespec giturl help gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push retire scratch-build sources spec \ srpm switch-branch tag unused-patches upload verify-files verrel" # parse main options and get command @@ -206,6 +206,9 @@ _rpkg() sources) options_dir="--outdir" ;; + spec) + options="--sources --print" + ;; srpm) options="--md5" ;; diff --git a/etc/rpkg/rpkg.conf b/etc/rpkg/rpkg.conf index c3c71be..1e4a085 100644 --- a/etc/rpkg/rpkg.conf +++ b/etc/rpkg/rpkg.conf @@ -11,3 +11,4 @@ kojiprofile = koji build_client = koji clone_config_rpms = bz.default-component %(module)s +preprocess_spec = False diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 52dcee3..382cf96 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -29,7 +29,10 @@ import tempfile import time from itertools import groupby from multiprocessing.dummy import Pool as ThreadPool +from tempfile import NamedTemporaryFile from operator import itemgetter +from munch import Munch +from pipes import quote import git import requests @@ -48,7 +51,7 @@ from pyrpkg.lookaside import CGILookasideCache from pyrpkg.sources import SourcesFile from pyrpkg.utils import (cached_property, extract_srpm, find_me, is_file_tracked, is_lookaside_eligible_file, - log_result) + log_result, run, macro_helper_cmd) from .gitignore import GitIgnore @@ -92,7 +95,7 @@ class Commands(object): build_client, user=None, dist=None, target=None, quiet=False, distgit_namespaced=False, realms=None, lookaside_namespaced=False, - git_excludes=None): + git_excludes=None, preprocess_spec=False, base_output_path=None): """Init the object and some configuration details.""" # Path to operate on, most often pwd @@ -214,6 +217,18 @@ class Commands(object): self.git_excludes = git_excludes or [] # Layout setup self.layout = layout.build(self.path) + # preproc attributes + self._preproc_preprocess_spec = preprocess_spec + self._preproc_base_output_path = base_output_path + self._preproc_produce_sources = False + self._preproc_use_existing_outdir = False + self._preproc_no_regenerate = False + self._preproc_version_bump = False + self._preproc_release_bump = False + self._preproc_spec_content = None + self._preproc_spec_path = None + self._preproc_git_props = None + self._preproc_outdir = None # Define properties here # Properties allow us to "lazy load" various attributes, which also means @@ -785,6 +800,10 @@ class Commands(object): self.load_spec() return self._spec + @spec.setter + def spec(self, value): + self._spec = value + def load_spec(self): """This sets the spec attribute""" @@ -1361,6 +1380,312 @@ class Commands(object): } return giturl + ##################################################### + ### PREPROCESSING HELPERS START + ##################################################### + + def _preprocess_spec(self, produce_sources=True, + use_existing_outdir=False, + no_regenerate=False): + """Does nothing if preprocessing is disabled. + + Otherwise, runs preprocessing on the spec file + and updates the instance attributes accordingly. + + The new spec file will be generated into a separate + newly created directory and self.path and self.layout + will be updated to point there. + + :param bool produce_sources: whether source files should be generated into + the outdir (needed for further building) + :param bool use_existing_outdir: whether we should operate on a previously + created outdir instead of creating new one + :param bool no_regenerate: if the target outdir already contains a + previously generated spec file, do nothing. + In this mode, nothing will be produced even + if produce_sources is set to True. + """ + # check if preprocessing is enabled first + if not self._preproc_preprocess_spec: + return + + # preload repo information for the current self.path + # if not preloaded yet + if not self._repo: + self.load_repo() + + # set the preprocessing flags + self._preproc_produce_sources = produce_sources + self._preproc_use_existing_outdir = use_existing_outdir + self._preproc_no_regenerate = no_regenerate + + # do the job + self.spec = self._final_spec() + + # update context + self.layout = layout.build(self._outdir) + self._path = self._outdir + + # load rpmdefines now after path is updated + self.load_rpmdefines() + + @property + def _outdir(self): + if self._preproc_outdir: + return self._preproc_outdir + + dirname_base = os.path.splitext( + os.path.splitext(os.path.basename(self._spec_path))[0])[0] + + cnt = len(glob.glob(os.path.join(self._preproc_base_output_path, dirname_base+'*'))) + + if not self._preproc_use_existing_outdir: + prefix = dirname_base + '-' + str(cnt+1) + '-' + + if not os.path.isdir(self._preproc_base_output_path): + prev_umask = os.umask(0) + os.makedirs(self._preproc_base_output_path, mode=0o1777) + os.umask(prev_umask) + + self._preproc_outdir = tempfile.mkdtemp(dir=self._preproc_base_output_path, prefix=prefix) + return self._preproc_outdir + + try: + prefix = dirname_base + '-' + str(cnt) + '-' + self._preproc_outdir = glob.glob(os.path.join(self._preproc_base_output_path, prefix+'*'))[0] + return self._preproc_outdir + except IndexError: + raise rpkgError('Could not load the latest outdir with prefix "%s" at %s' + % (prefix, self._preproc_base_output_path)) + + @property + def _spec_content(self): + if not self._preproc_spec_content: + with open(self._spec_path, 'r') as f: + self._preproc_spec_content = f.read() + return self._preproc_spec_content + + @property + def _spec_path(self): + if not self._preproc_spec_path: + self._preproc_spec_path = self._locate_spec(self.path) + return self._preproc_spec_path + + @property + def _git_props(self): + if self._preproc_git_props: + return self._preproc_git_props + + git_props = Munch() + + git_props.root = run( + macro_helper_cmd('git_root'), + capture_stderr=True, throw=False).out + + git_props.branch = run( + macro_helper_cmd('git_branch'), + env={'GIT_ROOT': git_props.root}, + capture_stderr=True, throw=False).out + + git_props.remote = run( + macro_helper_cmd('git_remote'), + env={'GIT_ROOT': git_props.root, 'GIT_BRANCH': git_props.branch}, + capture_stderr=True, throw=False).out + + git_props.remote_url = run( + macro_helper_cmd('git_remote_url'), + env={'GIT_ROOT': git_props.root, 'GIT_REMOTE': git_props.remote}, + capture_stderr=True, throw=False).out + + git_props.head = run( + macro_helper_cmd('git_head'), + env={'GIT_ROOT': git_props.root}, + capture_stderr=True, throw=False).out + + git_props.head_short = run( + macro_helper_cmd('git_head_short'), + env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head}, + capture_stderr=True, throw=False).out + + git_props.status = run( + macro_helper_cmd('git_status'), + env={'GIT_ROOT': git_props.root}, + capture_stderr=True, throw=False).out + + git_props.merged_tag_refs = run( + macro_helper_cmd('git_merged_tag_refs'), + env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head}, + capture_stderr=True, throw=False).out + + git_props.submodule_refs = run( + macro_helper_cmd('git_submodule_refs'), + env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head}, + capture_stderr=True, throw=False).out + + git_props.remote_netloc = urllib.parse.urlparse(git_props.remote_url).netloc + git_props.dirty = 'yes' if (git_props.status or ( + git_props.root and not git_props.head)) else '' + + self._preproc_git_props = git_props + return self._preproc_git_props + + def _preproc_source_params(self): + return '-s /usr/lib/rpkg.macros.d/git.bash' + + def _preproc_env_params(self, spec_templ): + spec_templ_dir = os.path.dirname(spec_templ) + + env_params = '-e INPUT_PATH={0} -e INPUT_DIR_PATH={1}'.format( + quote(spec_templ), quote(spec_templ_dir)) + + if self._preproc_produce_sources: + env_params += ' -e OUTDIR={0}'.format(quote(self._outdir)) + + if self._preproc_version_bump: + env_params += ' -e VERSION_BUMP=1' + + if self._preproc_release_bump: + env_params += ' -e RELEASE_BUMP=1' + + if self.verbose: + env_params += ' -e VERBOSE=1' + + return env_params + + def _preproc_git_props_params(self, status_file, + merged_tag_refs_file, + submodule_refs_file): + env_params = '' + + merged_tag_refs_file.write(self._git_props.merged_tag_refs+'\n') + submodule_refs_file.write(self._git_props.submodule_refs+'\n') + status_file.write(self._git_props.status+'\n') + + merged_tag_refs_file.close() + submodule_refs_file.close() + status_file.close() + + for key, val in self._git_props.items(): + if key == 'merged_tag_refs': + env_params += ' -e GIT_MERGED_TAG_REFS_FILE={0}'.format( + quote(merged_tag_refs_file.name)) + elif key == 'submodule_refs': + env_params += ' -e GIT_SUBMODULE_REFS_FILE={0}'.format( + quote(submodule_refs_file.name)) + elif key == 'status': + env_params += ' -e GIT_STATUS_FILE={0}'.format( + quote(status_file.name)) + else: + env_params += ' -e GIT_{0}={1}'.format( + key.upper(), quote(val)) + + return env_params.strip() + + def _locate_spec(cls, path): + spec_path = None + + for f in os.listdir(path): + if (f.endswith('.spec') or f.endswith('.spec.rpkg')) and not f.startswith('.'): + spec_path = os.path.abspath(os.path.join(path, f)) + break + + if not spec_path: + raise rpkgError('No spec file found at {0}.' + .format(path)) + return spec_path + + def _final_spec_text(self): + if not self._preproc_preprocess_spec: + return self._spec_content + + source_params = self._preproc_source_params() + env_params = self._preproc_env_params(self._spec_path) + + tmp0_file = NamedTemporaryFile( + 'w', prefix='tmp_', dir='/tmp/', delete=False) + tmp1_file = NamedTemporaryFile( + 'w', prefix='tmp_', dir='/tmp/', delete=False) + tmp2_file = NamedTemporaryFile( + 'w', prefix='tmp_', dir='/tmp/', delete=False) + tmp3_file = NamedTemporaryFile( + 'w', prefix='tmp_', dir='/tmp/', delete=False) + + git_props_params = self._preproc_git_props_params( + tmp1_file, tmp2_file, tmp3_file) + + tmp0_file.write(self._spec_content) + tmp0_file.close() + + cmd = 'cat {0} | preproc -C {1} {2} {3} {4}'.format( + quote(tmp0_file.name), quote(self.path), source_params, + env_params, git_props_params) + + try: + final_spec_text = run(cmd, shell=True, capture=True).out + finally: + os.unlink(tmp0_file.name) + os.unlink(tmp1_file.name) + os.unlink(tmp2_file.name) + os.unlink(tmp3_file.name) + + return final_spec_text + + def _final_spec_path(self): + final_spec_basename = os.path.basename( + self._spec_path[:-5] if self._spec_path.endswith('.rpkg') else self._spec_path) + return os.path.join(self._outdir, final_spec_basename) + + def _final_spec(self): + if not self._preproc_preprocess_spec: + return self.spec + + final_spec_path = self._final_spec_path() + + if os.path.exists(final_spec_path) and self._preproc_no_regenerate: + return self.spec + + with open(final_spec_path, 'w') as f: + f.write(self._final_spec_text()+'\n') + + if self._preproc_produce_sources: + self._setup_source_symlinks(final_spec_path) + + log.info('Wrote: {0}'.format(final_spec_path)) + return final_spec_path + + def _final_tmp_spec(self): + final_tmp_spec = NamedTemporaryFile( + 'w', prefix='tmp_', + dir='/tmp/', + delete=False) + + self._preproc_produce_sources = False + final_tmp_spec.write(self._final_spec_text()) + final_tmp_spec.close() + + return final_tmp_spec.name + + def _setup_source_symlinks(self, final_spec_path): + ts = rpm.ts() + + try: + rpm_spec = ts.parseSpec(final_spec_path) + except ValueError as e: + raise rpkgError("Could not parse spec file with error: "+str(e)) + + for (filename, num, flags) in rpm_spec.sources: + filename = os.path.basename(filename) + symlink_source = os.path.join(self.path, filename) + symlink_dest = os.path.join(self._outdir, filename) + + if os.path.isfile(symlink_source) and not \ + os.path.isfile(symlink_dest): + os.symlink(symlink_source, symlink_dest) + + ##################################################### + ### PREPROCESSING HELPERS END + ##################################################### + def add_tag(self, tagname, force=False, message=None, file=None): """Add a git tag to the repository @@ -2341,6 +2666,12 @@ class Commands(object): def clog(self, raw=False): """Write the latest spec changelog entry to a clog file""" + # remember orig path to write the "clog" file there + # becase preprocessing changes it if enabled + orig_path = self.path + + self._preprocess_spec(produce_sources=False) + spec_file = os.path.join(self.path, self.spec) # TODO: remove when fixed # Command contains workaround (undefines _changelog_trimtime) described at: @@ -2374,7 +2705,7 @@ class Commands(object): buf.close() # Now open the clog file and write out the lines - with open(os.path.join(self.path, 'clog'), 'w') as clog: + with open(os.path.join(orig_path, 'clog'), 'w') as clog: clog.writelines(clog_lines) def compile(self, arch=None, short=False, builddir=None, nocheck=False, @@ -2387,6 +2718,8 @@ class Commands(object): Logs the output and returns nothing """ + self._preprocess_spec(produce_sources=True) + # setup the rpm command cmd = ['rpmbuild'] cmd.extend(self.rpmdefines) @@ -2463,6 +2796,8 @@ class Commands(object): Parameter buildrootdir. """ + self._preprocess_spec(produce_sources=True) + # setup the rpm command cmd = ['rpmbuild'] cmd.extend(self.rpmdefines) @@ -2500,6 +2835,10 @@ class Commands(object): specified by the command line argument. """ + self._preprocess_spec(produce_sources=False, + use_existing_outdir=True, + no_regenerate=True) + # Check for srpm srpm = "%s-%s-%s.src.rpm" % (self.repo_name, self.ver, self.rel) if not os.path.exists(os.path.join(self.path, srpm)): @@ -2563,6 +2902,8 @@ class Commands(object): Parameter buildrootdir. """ + self._preprocess_spec(produce_sources=True) + # This could really use a list of arches to build for and loop over # Get the sources # build up the rpm command @@ -2872,6 +3213,8 @@ class Commands(object): Parameter buildrootdir. """ + self._preprocess_spec(produce_sources=True) + # setup the rpm command cmd = ['rpmbuild'] cmd.extend(self.rpmdefines) @@ -2914,6 +3257,8 @@ class Commands(object): the rpmbuild command. """ + self._preprocess_spec(produce_sources=True) + self.srpmname = os.path.join(self.path, "%s-%s-%s.src.rpm" % (self.repo_name, self.ver, self.rel)) @@ -3025,6 +3370,10 @@ class Commands(object): Parameter buildrootdir. """ + self._preprocess_spec(produce_sources=False, + use_existing_outdir=True, + no_regenerate=True) + # setup the rpm command cmd = ['rpmbuild'] cmd.extend(self.rpmdefines) diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index e6ba0c6..726643f 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -259,6 +259,9 @@ class cliClient(object): for excl in items.get("git_excludes", '').split('\n') if excl] + # find out if preprocessing is enabled + preprocess_spec = self._get_bool_opt('preprocess_spec') + # Create the cmd object self._cmd = self.site.Commands(self.args.path, items['lookaside'], @@ -276,7 +279,9 @@ class cliClient(object): distgit_namespaced=dg_namespaced, realms=realms, lookaside_namespaced=la_namespaced, - git_excludes=git_excludes + git_excludes=git_excludes, + preprocess_spec=preprocess_spec, + base_output_path='/tmp/{0}'.format(self.name) ) if self.args.module_name: @@ -471,6 +476,7 @@ class cliClient(object): self.register_retire() self.register_scratch_build() self.register_sources() + self.register_spec() self.register_srpm() self.register_switch_branch() self.register_tag() @@ -1406,6 +1412,25 @@ class cliClient(object): help='Directory to download files into (defaults to pwd)') sources_parser.set_defaults(command=self.sources) + def register_spec(self): + """Register the spec target""" + + spec_parser = self.subparsers.add_parser( + 'spec', help='Generate spec file from a spec template', + description='Preprocess a given spec template ' + 'and put the resulting spec file into outdir. The spec ' + 'template is simply copied into the outdir if ' + 'spec preprocessing is turned off or if the input spec ' + 'template does not contain any preprocessor macros.') + spec_parser.add_argument( + '--sources', action='store_true', default=False, + help='Enable source generation by the input ' + 'spec template. By default disabled.') + spec_parser.add_argument( + '--print', '-p', action='store_true', default=False, + help='Print rendered spec file to stdout.') + spec_parser.set_defaults(command=self.spec) + def register_srpm(self): """Register the srpm target""" @@ -1735,7 +1760,8 @@ class cliClient(object): if self.args.srpm == 'CONSTRUCT': self.log.debug('Generating an srpm') self.srpm() - self.args.srpm = '%s.src.rpm' % self.cmd.nvr + srpm_path = os.path.join(self.cmd.path, '%s.src.rpm' % self.cmd.nvr) + self.args.srpm = srpm_path return self._upload_file_for_build(self.args.srpm) def _watch_build_tasks(self, task_ids): @@ -2099,9 +2125,9 @@ class cliClient(object): self.log.debug('Generating an srpm') self.args.hash = None self.srpm() - srpm_name = '%s.src.rpm' % self.cmd.nvr + srpm_path = os.path.join(self.cmd.path, '%s.src.rpm' % self.cmd.nvr) self.cmd.copr_build(self.args.project[0], - srpm_name, + srpm_path, self.args.nowait, self.args.copr_config) @@ -2598,6 +2624,15 @@ class cliClient(object): outdir = getattr(self.args, 'outdir', None) self.cmd.sources(outdir) + def spec(self): + self.cmd._preproc_produce_sources = self.args.sources + + if self.args.print: + final_spec_text = self.cmd._final_spec_text() + print(final_spec_text) + else: + self.cmd._final_spec() + def srpm(self): self.sources() diff --git a/pyrpkg/utils.py b/pyrpkg/utils.py index d754d78..d314172 100644 --- a/pyrpkg/utils.py +++ b/pyrpkg/utils.py @@ -18,6 +18,12 @@ import argparse import os import subprocess import sys +import munch +import tempfile +import textwrap +import logging +import pipes +import re import git import six @@ -33,6 +39,9 @@ else: getcwd = os.getcwdu +from pyrpkg.errors import rpkgError + +log = logging.getLogger("pyrpkg") class cached_property(property): """A property caching its return value @@ -294,3 +303,73 @@ def is_lookaside_eligible_file(file_name, dir_path=None): # output contains encoding ("binary", "us-ascii", ...) encoding = output.strip() # strip newline at the end return encoding == "binary" + +##################################################### +### PREPROCESSING UTILS START +##################################################### + +MACRO_HELPER_DIR = '/usr/lib/rpkg.macros.d/helpers/' + +def run(cmd, shell=False, env=None, cwd=None, capture=True, capture_stderr=False, throw=True): + if capture: + stdout = subprocess.PIPE + else: + stdout = None + + if capture_stderr: + stderr = subprocess.PIPE + else: + stderr = None + + log.debug('Running: {0}'.format( + cmd_repr(cmd) if not shell else cmd)) + + process = subprocess.Popen( + cmd, stdout=stdout, stderr=stderr, shell=shell, env=env, cwd=cwd) + + (out, err) = process.communicate() + + err = err.decode('utf-8').strip('\n') if err else '' + + if process.returncode != 0 and throw: + raise rpkgError(err) + + out = out.decode('utf-8').strip('\n') if out else '' + + return munch.Munch(out=out, + err=err, + rc=process.returncode) + + +def changelog_entry(commit_message): + entry = '- ' + commit_message.split('\n')[0] if commit_message else '' + return '\n'.join(textwrap.wrap(re.sub(r'(\s+)%([^%])', r'\1%%\2', entry), 80)) + + +def edit(editor, text): + tmp_f = tempfile.NamedTemporaryFile('w', delete=False) + tmp_f.write(text) + tmp_f.close() + subprocess.check_call(editor.split()+[tmp_f.name]) + tmp_f = open(tmp_f.name, 'r') + result = tmp_f.read() + tmp_f.close() + os.unlink(tmp_f.name) + return result + + +def cmd_repr(cmd): + quoted_items = [] + + for i in range(len(cmd)): + quoted_items.append(pipes.quote(cmd[i])) + + return ' '.join(quoted_items) + + +def macro_helper_cmd(*args): + return [MACRO_HELPER_DIR+args[0], *args[1:]] + +##################################################### +### PREPROCESSING UTILS END +##################################################### diff --git a/requirements/fedora-cli-tools.txt b/requirements/fedora-cli-tools.txt index 5209cc7..11d894d 100644 --- a/requirements/fedora-cli-tools.txt +++ b/requirements/fedora-cli-tools.txt @@ -5,3 +5,5 @@ mock rpm-build rpmlint module-build-service # needed for the module-buld-local command +rpkg-macros +preproc diff --git a/requirements/fedora-py2.txt b/requirements/fedora-py2.txt index 58b994d..bd1666d 100644 --- a/requirements/fedora-py2.txt +++ b/requirements/fedora-py2.txt @@ -10,6 +10,7 @@ python2-requests PyYAML # python2-openidc-client # used for MBS OIDC authentication # python2-requests-kerberos # used for MBS Kerberos authentication +python2-munch # For running tests python2-coverage diff --git a/requirements/fedora-py3.txt b/requirements/fedora-py3.txt index fc799c3..e9ddcdd 100644 --- a/requirements/fedora-py3.txt +++ b/requirements/fedora-py3.txt @@ -10,6 +10,7 @@ python3-requests python3-yaml # python3-openidc-client # used for MBS OIDC authentication # python3-requests-kerberos # used for MBS Kerberos authentication +python3-munch # For running tests python3-coverage diff --git a/requirements/pypi.txt b/requirements/pypi.txt index 6aeaee2..73bf53f 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -9,6 +9,7 @@ pycurl >= 7.19 six >= 1.9.0 requests PyYAML +munch # rpm-py-installer # diff --git a/tests/test_cli.py b/tests/test_cli.py index cdbc241..c3772ef 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1663,10 +1663,6 @@ class TestCoprBuild(CliTestCase): super(TestCoprBuild, self).tearDown() def assert_copr_build(self, cli_cmd, expected_copr_cli): - with patch('sys.argv', new=cli_cmd): - cli = self.new_cli() - cli.copr_build() - self.mock_srpm.assert_called_once() self.mock_run_command.assert_called_once_with(expected_copr_cli) @@ -1674,18 +1670,26 @@ class TestCoprBuild(CliTestCase): cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'copr-build', 'user/project'] + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.copr_build() + self.assert_copr_build(cli_cmd, [ 'copr-cli', 'build', 'user/project', - '{0}.src.rpm'.format(self.mock_nvr.return_value) + os.path.join(cli.cmd.path, '{0}.src.rpm'.format(self.mock_nvr.return_value)) ]) def test_copr_build_no_wait(self): cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'copr-build', '--nowait', 'user/project'] + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.copr_build() + self.assert_copr_build(cli_cmd, [ 'copr-cli', 'build', '--nowait', 'user/project', - '{0}.src.rpm'.format(self.mock_nvr.return_value) + os.path.join(cli.cmd.path, '{0}.src.rpm'.format(self.mock_nvr.return_value)) ]) def test_copr_build_with_alternative_config_file(self): @@ -1693,10 +1697,14 @@ class TestCoprBuild(CliTestCase): 'copr-build', '--config', '/path/to/alternative/config', 'user/project'] + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.copr_build() + self.assert_copr_build(cli_cmd, [ 'copr-cli', '--config', '/path/to/alternative/config', 'build', 'user/project', - '{0}.src.rpm'.format(self.mock_nvr.return_value) + os.path.join(cli.cmd.path, '{0}.src.rpm'.format(self.mock_nvr.return_value)) ]) @@ -3439,7 +3447,7 @@ class TestBuildPackage(FakeKojiCreds, CliTestCase): srpm_file, unique_path = args if expected_srpm_file is None: - self.assertEqual('{0}.src.rpm'.format(cli.cmd.nvr), srpm_file) + self.assertEqual(os.path.join(cli.cmd.path, '{0}.src.rpm'.format(cli.cmd.nvr)), srpm_file) else: self.assertEqual(expected_srpm_file, srpm_file) six.assertRegex(self, unique_path, r'^cli-build/\d+\.\d+\.[a-zA-Z]+$') From e2ffb1da07e8e36ef3b11f6cd7c3836c489ec69e Mon Sep 17 00:00:00 2001 From: clime Date: Jan 15 2021 19:41:51 +0000 Subject: [PATCH 2/4] fix return value from _final_spec when no_regenerate and spec file exists Signed-off-by: clime --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 382cf96..ded91fc 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -1642,7 +1642,7 @@ class Commands(object): final_spec_path = self._final_spec_path() if os.path.exists(final_spec_path) and self._preproc_no_regenerate: - return self.spec + return final_spec_path with open(final_spec_path, 'w') as f: f.write(self._final_spec_text()+'\n') From 835cf9dfa594d05deead001e4576c63a1e818a46 Mon Sep 17 00:00:00 2001 From: clime Date: Jan 15 2021 19:41:51 +0000 Subject: [PATCH 3/4] fix macro_helper_cmd for python versions < 3.5 Signed-off-by: clime --- diff --git a/pyrpkg/utils.py b/pyrpkg/utils.py index d314172..bb987e5 100644 --- a/pyrpkg/utils.py +++ b/pyrpkg/utils.py @@ -368,7 +368,7 @@ def cmd_repr(cmd): def macro_helper_cmd(*args): - return [MACRO_HELPER_DIR+args[0], *args[1:]] + return [MACRO_HELPER_DIR+args[0]] + list(args[1:]) ##################################################### ### PREPROCESSING UTILS END From dd3fa1fd0b2d84b97f178a0b6bcef14a35d38a4a Mon Sep 17 00:00:00 2001 From: clime Date: Jan 15 2021 21:22:45 +0000 Subject: [PATCH 4/4] add support for verrel command Signed-off-by: clime --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index ded91fc..37e1235 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -229,6 +229,7 @@ class Commands(object): self._preproc_spec_path = None self._preproc_git_props = None self._preproc_outdir = None + self._preproc_done = False # Define properties here # Properties allow us to "lazy load" various attributes, which also means @@ -691,13 +692,20 @@ class Commands(object): def load_nameverrel(self): """Set the release of a package.""" + tmp_spec_created = False + if self._preproc_preprocess_spec and not self._preproc_done: + spec = self._final_tmp_spec() + tmp_spec_created = True + else: + spec = self.spec + cmd = ['rpm'] cmd.extend(self.rpmdefines) # We make sure there is a space at the end of our query so that # we can split it later. When there are subpackages, we get a # listing for each subpackage. We only care about the first. cmd.extend(['-q', '--qf', '"??%{NAME} %{EPOCH} %{VERSION} %{RELEASE}??"', - '--specfile', '"%s"' % os.path.join(self.path, self.spec)]) + '--specfile', '"%s"' % os.path.join(self.path, spec)]) joined_cmd = ' '.join(cmd) try: proc = subprocess.Popen(joined_cmd, shell=True, @@ -712,13 +720,17 @@ class Commands(object): self.log.error(err) raise rpkgError('Could not query n-v-r of %s: %s' % (self.repo_name, e)) + finally: + if tmp_spec_created: + os.remove(spec) + if err: self.log.debug('Errors occoured while running following command to get N-V-R-E:') self.log.debug(joined_cmd) self.log.error(err) if proc.returncode > 0: raise rpkgError('Could not get n-v-r-e from %s' - % os.path.join(self.path, self.spec)) + % os.path.join(self.path, spec)) # Get just the output, then split it by ??, grab the first and split # again to get ver and rel @@ -1429,6 +1441,9 @@ class Commands(object): # load rpmdefines now after path is updated self.load_rpmdefines() + # make note that spec was already preprocessed + self._preproc_done = True + @property def _outdir(self): if self._preproc_outdir: @@ -1656,7 +1671,7 @@ class Commands(object): def _final_tmp_spec(self): final_tmp_spec = NamedTemporaryFile( 'w', prefix='tmp_', - dir='/tmp/', + dir='/tmp', delete=False) self._preproc_produce_sources = False