From d4d22461c9b4aa4bddc6c1332e8a8050239e6c9c Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 11 2017 11:00:55 +0000 Subject: [PATCH 1/2] [backend] unless reattaching, don't pgrep for mockchain pid --- diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index f2e0eda..134d2ca 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -37,6 +37,7 @@ class Builder(object): self.conn = SSHConnection(user=self.opts.build_user, host=self.hostname, config_file=self.opts.ssh.builder_config) self.module_dist_tag = self._load_module_dist_tag() + self._build_pid = None def _load_module_dist_tag(self): module_md_filepath = os.path.join(self.job.destdir, self.job.chroot, "module_md.yaml") @@ -260,10 +261,10 @@ class Builder(object): # on std{out,err} which means that it's output are not line-buffered # (which wouldn't be very useful live-log), so let's use `unbuffer` # from expect.rpm to allocate _persistent_ server-side pseudo-terminal - buildcmd_async = 'trap "" SIGHUP; unbuffer {buildcmd} &>{livelog} &'.format( + buildcmd_async = 'trap "" SIGHUP; unbuffer {buildcmd} &>{livelog} & echo !$'.format( livelog=self.livelog_name, buildcmd=buildcmd) - - self._run_ssh_cmd(buildcmd_async) + pid, _ = self._run_ssh_cmd(buildcmd_async) + self._build_pid =int(pid.strip()) def setup_pubsub_handler(self): @@ -298,12 +299,22 @@ class Builder(object): # buildcmd = self.gen_mockchain_command(dest) # - def attach_to_build(self): + @property + def build_pid(self): + if self._build_pid: + return self._build_pid try: pidof_cmd = "/usr/bin/pgrep -o -u {user} {command}".format( user=self.opts.build_user, command="mockchain") out, _ = self._run_ssh_cmd(pidof_cmd) - except RemoteCmdError: + except: + return None + + return int(out.strip()) + + + def attach_to_build(self): + if not self.build_pid: self.log.info("Build is not running. Continuing...") return @@ -311,7 +322,7 @@ class Builder(object): live_log = os.path.join(self.job.results_dir, 'mockchain-live.log') live_cmd = '/usr/bin/tail -f --pid={pid} {log}'.format( - pid=out.strip(), log=self.livelog_name) + pid=self.build_pid, log=self.livelog_name) self.log.info("Attaching to live build log: " + live_cmd) with open(live_log, 'w') as logfile: From 0fbb08f317f360b38cb164c4ac6e516dc4f66bb4 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Apr 11 2017 11:00:55 +0000 Subject: [PATCH 2/2] [backend] standalone builder When copr-be.conf contains this snippet .. [builder] standalone=true config=/etc/copr-builder/.conf .. then instead of controlling all the build peculiarities manually from backend, the build is fully controlled builder-side, while the build process logic is concentrated in one, easily reproducible command: copr-builder --config /tmp/copr.conf [opts] --detached This extremely simplifies debugging when something goes wrong, but it is also the first step towards "heterogeneous sets of builders" feature https://bugzilla.redhat.com/show_bug.cgi?id=1334701 The builder VMs just need to have installed package 'copr-builder'. --- diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index f5023eb..04cbc20 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -287,6 +287,12 @@ class BackendConfigReader(object): opts.consecutive_failure_threshold = _get_conf( cp, "builder", "consecutive_failure_threshold", DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int") + opts.standalone_builder = _get_conf( + cp, "builder", "standalone", False, mode="bool") + opts.standalone_builder_config = _get_conf( + cp, "builder", "config", "/etc/copr-builder/fedora-copr.conf") + + opts.log_dir = _get_conf( cp, "backend", "log_dir", "/var/log/copr-backend/") opts.log_level = _get_conf( diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 134d2ca..52288e3 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -66,9 +66,12 @@ class Builder(object): if self._remote_tempdir: return self._remote_tempdir - tempdir_path = "{0}/{1}-{2}".format( - self._remote_basedir, "mockremote", self.job.task_id) - self._run_ssh_cmd("/bin/mkdir -m 755 -p {0}".format(tempdir_path)) + if self.opts.standalone_builder: + tempdir_path = "/var/lib/copr-builder" + else: + tempdir_path = "{0}/{1}-{2}".format( + self._remote_basedir, "mockremote", self.job.task_id) + self._run_ssh_cmd("/bin/mkdir -m 755 -p {0}".format(tempdir_path)) self._remote_tempdir = tempdir_path return self._remote_tempdir @@ -99,6 +102,9 @@ class Builder(object): return out, err def _get_remote_results_dir(self): + if self.opts.standalone_builder: + return os.path.join(self.tempdir, 'results') + if any(x is None for x in [self.remote_build_dir, self.remote_pkg_name, self.job.chroot]): @@ -233,9 +239,18 @@ class Builder(object): @property def livelog_name(self): + if self.opts.standalone_builder: + return "/var/lib/copr-builder/live-log" return pipes.quote('/tmp/{}.log'.format(self.job.task_id)) - def run_mockchain_async(self): + def run_async_build(self): + if self.opts.standalone_builder: + cmd = self._copr_builder_cmd() + pid, _ = self._run_ssh_cmd(cmd) + self._build_pid =int(pid.strip()) + return + + buildcmd = "timeout {} {} -r {} -l {} ".format( self.timeout, mockchain, pipes.quote(self.get_chroot_config_path(self.job.chroot)), pipes.quote(self.remote_build_dir)) @@ -306,6 +321,10 @@ class Builder(object): try: pidof_cmd = "/usr/bin/pgrep -o -u {user} {command}".format( user=self.opts.build_user, command="mockchain") + + if self.opts.standalone_builder: + pidof_cmd = "cat /var/lib/copr-builder/pid" + out, _ = self._run_ssh_cmd(pidof_cmd) except: return None @@ -313,6 +332,28 @@ class Builder(object): return int(out.strip()) + def _copr_builder_cmd(self): + template = 'copr-builder --config {config} --copr {copr} ' \ + + '--package {package} --revision {revision} ' \ + + '--host-resolv {net} --timeout {timeout} ' \ + + '--chroot {chroot} --detached ' + + # Repo name like // + git_repo_path = self.job.git_repo.split('/') + + copr = '{0}/{1}'.format(git_repo_path[0], git_repo_path[1]) + package = git_repo_path[2] + + return template.format( + config=self.opts.standalone_builder_config, + copr=copr, + package=package, + revision=self.job.git_hash, + net="True" if self.job.enable_net else "False", + chroot=self.job.chroot, + timeout=self.timeout, + ) + def attach_to_build(self): if not self.build_pid: self.log.info("Build is not running. Continuing...") @@ -321,7 +362,7 @@ class Builder(object): ensure_dir_exists(self.job.results_dir, self.log) live_log = os.path.join(self.job.results_dir, 'mockchain-live.log') - live_cmd = '/usr/bin/tail -f --pid={pid} {log}'.format( + live_cmd = '/usr/bin/tail -n +0 -f --pid={pid} {log}'.format( pid=self.build_pid, log=self.livelog_name) self.log.info("Attaching to live build log: " + live_cmd) @@ -331,14 +372,15 @@ class Builder(object): def build(self): - # make mock config - self.setup_mock_chroot_config() + if not self.opts.standalone_builder: + # make mock config + self.setup_mock_chroot_config() - # download the package to the builder - self.download_job_pkg_to_builder() + # download the package to the builder + self.download_job_pkg_to_builder() # run the build - self.run_mockchain_async() + self.run_async_build() # attach to building output self.attach_to_build() @@ -386,6 +428,11 @@ class Builder(object): self.rsync_call(self._get_remote_config_dir(), target_path) def check(self): + if self.opts.standalone_builder: + # We simply expect that 'copr-builder' package is installed on the + # builder machine. + return + # do check of host try: # requires name resolve facility