#2371 Upgrade docker images to F37
Merged 2 years ago by praiskup. Opened 2 years ago by frostyx.
copr/ frostyx/copr docker-f37  into  main

@@ -5,7 +5,7 @@ 

  import glob

  import logging

  import os

- import pipes

+ import shlex

  import shutil

  import statistics

  import time
@@ -645,7 +645,7 @@ 

              "builtin cd {0} && "

              "for f in `ls *.rpm | grep -v \"src.rpm$\"`; do"

              "   rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; "

-             "done".format(pipes.quote(job.results_dir))

+             "done".format(shlex.quote(job.results_dir))

          )

  

          result = run_cmd(cmd, shell=True, logger=self.log)

@@ -8,7 +8,7 @@ 

  import time

  import types

  import glob

- import pipes

+ import shlex

  

  import configparser

  from configparser import ConfigParser
@@ -111,7 +111,7 @@ 

      pid = process.pid

      str_cmd = cmd

      if isinstance(cmd, list):

-         str_cmd = " ".join([pipes.quote(a) for a in cmd])

+         str_cmd = " ".join([shlex.quote(a) for a in cmd])

  

      _info(logger, "Running command '%s' as PID %s", str_cmd, pid)

  

@@ -86,7 +86,7 @@ 

          output from particular command is ignored.

  

          :param user_command:

-             Command (string) to be executed (note: use pipes.quote).

+             Command (string) to be executed (note: use shlex.quote).

  

          :param max_retries:

              When there is ssh connection problem, re-try the action at most
@@ -122,7 +122,7 @@ 

          waste a lot of memory, run() is better option.

  

          :param user_command:

-             Command (string) to be run as string (note: use pipes.quote).

+             Command (string) to be run as string (note: use shlex.quote).

  

          :param max_retries:

              When there is ssh connection problem, re-try the action at most

@@ -1,4 +1,4 @@ 

- FROM fedora:36

+ FROM fedora:37

  ARG COPR_PACKAGES=devel

  MAINTAINER copr-devel@lists.fedorahosted.org

  ENV container docker

@@ -20,8 +20,8 @@ 

  # Some tests might want to install built packages

  # Therefore, these packages need to be built for the same fedora version

  # as this script is going to be run from

- FEDORA_VERSION=36

- PREV_FEDORA_VERSION=35

+ FEDORA_VERSION=37

+ PREV_FEDORA_VERSION=36

  

  CHROOT="fedora-$FEDORA_VERSION-x86_64"

  PREV_CHROOT="fedora-$FEDORA_VERSION-x86_64"

file modified
+2 -2
@@ -3,7 +3,7 @@ 

  from contextlib import contextmanager

  import io

  import json

- import pipes

+ import shlex

  import subprocess

  import sys

  from urllib.parse import urlparse
@@ -24,7 +24,7 @@ 

  

  def quoted_cmd(cmd):

      """ shell quoted cmd array as string """

-     return " ".join(pipes.quote(arg) for arg in cmd)

+     return " ".join(shlex.quote(arg) for arg in cmd)

  

  

  def run(cmd):

@@ -47,12 +47,30 @@ 

  ``Click to launch`` button to launch an instance from the x86_64

  AMI. Select the US East (N. Virginia) region.

  

+ You will get redirected to the Amazon AWS page.

  

- 2. Choose Instance Type

- .......................

  

- You will get redirected to Amazon and asked to choose an instance

- type. Currently, we use the following:

+ 2. Name and tags

+ ................

+ 

+ - Set ``Name`` and add ``-new`` suffix (e.g. ``copr-distgit-dev-new``

+   or ``copr-distgit-prod-new``)

+ - Set ``CoprInstance`` to ``devel`` or ``production``

+ - Set ``CoprPurpose`` to ``infrastructure``

+ - Set ``FedoraGroup`` to ``copr``

+ 

+ 

+ 3. Application and OS Images (Amazon Machine Image)

+ ...................................................

+ 

+ Skip this section, we already chose the correct AMI from the Fedora

+ website.

+ 

+ 

+ 4. Instance type

+ ................

+ 

+ Currently, we use the following instance types:

  

  +----------------+-------------+-------------+

  |                | Dev         | Production  |
@@ -70,56 +88,60 @@ 

  the cheapest available instance type according to our needs.

  

  

- 3. Configure Instance

- .....................

- 

- - Select ``Network`` without ``| foo`` suffix

- - Select ``Subnet`` to be ``us-east-1c``

- - Opt-in the ``Protect against accidental termination`` checkbox

- - Request IPv6 assignment ``IPv6 IPs -> Add IP``

+ 5. Key pair (login)

+ ...................

  

+ - Make sure to use existing key pair named ``Ansible Key``.  This allows us to

+   run the playbooks on ``batcave01`` box against the newly spawned VM.

  

- 4. Add Storage

- ..............

  

- - Update the ``Size (GiB)`` value to resemble root partition size of

-   the currently running instance

- - Select ``Encryption`` key, don't leave the partition

-   unencrypted. Use whatever is ``(default)``

+ 6. Network settings

+ ...................

  

+ - Click the ``Edit`` button in the box heading to show more options

+ - Select VPC ``vpc-0af***********972``

+ - Select ``Subnet`` to be ``us-east-1c``

+ - Switch ``Auto-assign IPv6 IP`` to ``Enable``

+ - Switch to ``Select existing security group`` and pick one of

  

- 5. Add Tags

- ...........

+     - ``copr-frontend-sg``

+     - ``copr-backend-sg``

+     - ``copr-distgit-sg``

+     - ``copr-keygen-sg``

  

- - Set ``CoprInstance`` to ``devel`` or ``production``

- - Set ``CoprPurpose`` to ``infrastructure``

- - Set ``FedoraGroup`` to ``copr``

- - Set ``Name`` and add ``-new`` suffix (e.g. ``copr-distgit-dev-new``

-   or ``copr-distgit-prod-new``)

  

+ 7. Configure storage

+ ....................

  

- 6. Configure Security Group

- ...........................

+ - Click the ``Advanced`` button in the box heading to show more options

+ - Update the ``Size (GiB)`` of the root partition

  

- - Select an existing security group and pick one of

+ +----------------+-------------+-------------+

+ |                | Dev         | Production  |

+ +================+=============+=============+

+ | **frontend**   | 50G         | 50G         |

+ +----------------+-------------+-------------+

+ | **backend**    | 20G         | 100G        |

+ +----------------+-------------+-------------+

+ | **keygen**     | 10G         | 20G         |

+ +----------------+-------------+-------------+

+ | **distgit**    | 20G         | 30G         |

+ +----------------+-------------+-------------+

  

-     - ``copr-frontend-sg``

-     - ``copr-backend-sg``

-     - ``copr-distgit-sg``

-     - ``copr-keygen-sg``

+ - Turn on the ``Encrypted`` option

+ - Select ``KMS key`` to whatever is ``(default)``

  

  

- 7. Review

- .........

+ 8. Advanced details

+ ...................

  

- ``Review and Launch`` the instance.

+ - ``Termination protection`` - ``Enable``

  

  

- 8. Public Key

- .............

+ 9. Launch instance

+ ..................

  

- - Make sure to use existing key pair named ``Ansible Key``.  This allows us to

-   run the playbooks on ``batcave01`` box against the newly spawned VM.

+ Click ``Launch instance`` in the right panel.

  

  

  Pre-prepare the new VM

@@ -7,6 +7,10 @@ 

  When upgrading to a distribution that provides a new major version of PostgreSQL server,

  there is a manual intervention required.

  

+ Make sure ``httpd`` is stopped::

+ 

+     [root@copr-fe-dev ~][STG]# systemctl stop httpd

+ 

  Upgrade the database::

  

      [root@copr-fe-dev ~][STG]# dnf install postgresql-upgrade
@@ -18,3 +22,7 @@ 

      [root@copr-fe-dev ~][STG]# su postgres

      bash-5.0$ cd

      bash-5.0$ reindexdb --all

+ 

+ Start ``httpd`` again::

+ 

+     [root@copr-fe-dev ~][STG]# systemctl start httpd

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  MAINTAINER copr-devel@lists.fedorahosted.org

  

  ENV export LANG=en_US.UTF-8

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  MAINTAINER copr-devel@lists.fedorahosted.org

  

  # TERM is to make the tito work in container, rhbz#1733043

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  MAINTAINER copr-devel@lists.fedorahosted.org

  

  # TERM is to make the tito work in container, rhbz#1733043

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  MAINTAINER copr-devel@lists.fedorahosted.org

  

  # TERM is to make the tito work in container, rhbz#1733043

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  

  MAINTAINER copr-devel@lists.fedorahosted.org

  

file modified
+1 -1
@@ -1,4 +1,4 @@ 

- FROM registry.fedoraproject.org/fedora:35

+ FROM registry.fedoraproject.org/fedora:37

  MAINTAINER copr-devel@lists.fedorahosted.org

  

  # Deployment instructions are described here

@@ -6,7 +6,7 @@ 

  import copy

  import logging

  from functools import wraps

- import pipes

+ import shlex

  import importlib

  import click

  from copr_common.log import setup_script_logger
@@ -46,7 +46,7 @@ 

  if os.getuid() == 0:

      sys.stderr.write("Please don't run this script as a 'root' user, use:\n")

      sys.stderr.write("$ sudo -u copr-fe {}\n".format(

-             ' '.join([pipes.quote(arg) for arg in sys.argv])))

+             ' '.join([shlex.quote(arg) for arg in sys.argv])))

      sys.exit(1)

  

  commands_list =	[

@@ -7,7 +7,7 @@ 

  import errno

  import glob

  import logging

- import pipes

+ import shlex

  import os

  import subprocess

  import sys
@@ -24,7 +24,7 @@ 

  

  def log_cmd(command, comment="Running command"):

      """ Dump the command to stderr so it can be c&p to shell """

-     command = ' '.join([pipes.quote(x) for x in command])

+     command = ' '.join([shlex.quote(x) for x in command])

      logging.info("%s: %s", comment, command)

  

  

@@ -7,7 +7,7 @@ 

  import re

  import configparser

  import datetime

- import pipes

+ import shlex

  from threading import Timer

  from collections import OrderedDict

  import rpm
@@ -33,7 +33,7 @@ 

  

  

  def cmd_readable(cmd):

-     return ' '.join([pipes.quote(part) for part in cmd])

+     return ' '.join([shlex.quote(part) for part in cmd])

  

  

  def run_cmd(cmd, cwd=".", preexec_fn=None):
@@ -223,7 +223,7 @@ 

  

  def dump_live_log(logfile):

      filter_continuing_lines = "/usr/bin/copr-rpmbuild-loggify"

-     tee_output = "tee -a {0}".format(pipes.quote(logfile))

+     tee_output = "tee -a {0}".format(shlex.quote(logfile))

      cmd = filter_continuing_lines + "|" + tee_output

      tee = subprocess.Popen(cmd, stdin=subprocess.PIPE, shell=True)

      os.dup2(tee.stdin.fileno(), sys.stdout.fileno())

file modified
+2 -2
@@ -9,7 +9,7 @@ 

  import logging

  import shutil

  import pprint

- import pipes

+ import shlex

  import pkg_resources

  

  try:
@@ -113,7 +113,7 @@ 

      with open(config.get("main", "logger_pidfile"), "w") as pidfile:

          pidfile.write(str(logging_pid))

  

-     log.info('Running: {0}'.format(" ".join(map(pipes.quote, sys.argv))))

+     log.info('Running: {0}'.format(" ".join(map(shlex.quote, sys.argv))))

I would probably prefer to do pipes => shlex move everywhere:

$ git grep pipes | grep import 
backend/copr_backend/background_worker_build.py:import pipes
backend/copr_backend/helpers.py:import pipes
backend/run/copr-backend-analyze-results:import pipes
backend/run/copr-backend-generate-graphs:import pipes
backend/run/copr-repo:import pipes
behave/copr_behave_lib.py:import pipes
frontend/coprs_frontend/manage.py:import pipes
rpmbuild/bin/copr-sources-custom:import pipes
rpmbuild/copr_distgit_client.py:import pipes
rpmbuild/copr_rpmbuild/helpers.py:import pipes
rpmbuild/main.py:import pipes

      log.info('Version: {0}'.format(VERSION))

      log.info("PID: %s", main_pid)

      log.info("Logging PID: %s", logging_pid)

Build succeeded.

1 new commit added

  • beaker-tests: upgrade DockerTestEnv to F37
2 years ago

Build succeeded.

I would probably prefer to do pipes => shlex move everywhere:

$ git grep pipes | grep import 
backend/copr_backend/background_worker_build.py:import pipes
backend/copr_backend/helpers.py:import pipes
backend/run/copr-backend-analyze-results:import pipes
backend/run/copr-backend-generate-graphs:import pipes
backend/run/copr-repo:import pipes
behave/copr_behave_lib.py:import pipes
frontend/coprs_frontend/manage.py:import pipes
rpmbuild/bin/copr-sources-custom:import pipes
rpmbuild/copr_distgit_client.py:import pipes
rpmbuild/copr_rpmbuild/helpers.py:import pipes
rpmbuild/main.py:import pipes

1 new commit added

  • doc: update the persistent instances documentation
2 years ago

Build succeeded.

rebased onto 16c11b3

2 years ago

Commit b4a21e0 fixes this pull-request

Pull-Request has been merged by praiskup

2 years ago

Commit e3d78b9 fixes this pull-request

Pull-Request has been merged by praiskup

2 years ago

Commit b5eac22 fixes this pull-request

Pull-Request has been merged by praiskup

2 years ago

Commit 3a47e3e fixes this pull-request

Pull-Request has been merged by praiskup

2 years ago

Commit dc1ab43 fixes this pull-request

Pull-Request has been merged by praiskup

2 years ago