PR#59: Modularity improvements - copr/copr

		`@@ -10,14 +10,13 @@`
		`import time`
		`import six`
		`import simplejson`
		`- import random`
		`from collections import defaultdict`

		`import logging`
		`if six.PY2:`
		`- from urlparse import urlparse, urlencode`
		`+ from urlparse import urlparse`
		`else:`
		`- from urllib.parse import urlparse, urlencode`
		`+ from urllib.parse import urlparse`

		`if sys.version_info < (2, 7):`
		`class NullHandler(logging.Handler):`
		`@@ -30,10 +29,10 @@`
		`log.addHandler(NullHandler())`

		`from copr import CoprClient`
		`+ from copr.client.responses import CoprResponse`
		`import copr.exceptions as copr_exceptions`

		`from .util import ProgressBar`
		`- from .util import listen_for_token`
		`from .build_config import MockProfile`


		`@@ -565,49 +564,10 @@`
		`"""`
		`Build module via Copr MBS`
		`"""`
		`- token = args.token`
		`- if not token:`
		`- print("Provide token as command line argument or visit following URL to obtain the token:")`
		`- query = urlencode({`
		`- 'response_type': 'token',`
		`- 'response_mode': 'form_post',`
		`- 'nonce': random.randint(100, 10000),`
		`- 'scope': ' '.join([`
		`- 'openid',`
		`- 'https://id.fedoraproject.org/scope/groups',`
		`- 'https://mbs.fedoraproject.org/oidc/submit-build',`
		`- ]),`
		`- 'client_id': 'mbs-authorizer',`
		`- }) + "&redirect_uri=http://localhost:13747/"`
		`- print("https://id.fedoraproject.org/openidc/Authorization?" + query)`
		`- print("We are waiting for you to finish the token generation...")`
		`-`
		`- token = listen_for_token()`
		`- print("Token: {}".format(token))`
		`- print("")`
		`- if not token:`
		`- print("Failed to get a token from response")`
		`- sys.exit(1)`
		`-`
		`+ ownername, projectname = parse_name(args.copr or "")`
		`modulemd = open(args.yaml, "rb") if args.yaml else args.url`
		`- response = self.client.build_module(modulemd, token)`
		`- if response.status_code == 500:`
		`- print(response.text)`
		`- sys.exit(1)`
		`-`
		`- data = response.json()`
		`- if response.status_code != 201:`
		`- print("Error: {}".format(data["message"]))`
		`- sys.exit(1)`
		`- print("Created module {}-{}-{}".format(data["name"], data["stream"], data["version"]))`
		`-`
		`- def action_make_module(self, args):`
		`- """`
		`- Fake module build`
		`- """`
		`- ownername, projectname = parse_name(args.copr)`
		`- result = self.client.make_module(projectname, args.yaml, username=ownername)`
		`- print(result.message if result.output == "ok" else result.error)`
		`+ response = self.client.build_module(modulemd, ownername, projectname)`
		`+ print(response.message if response.output == "ok" else response.error)`


		`def setup_parser():`
		`@@ -994,20 +954,12 @@`

		`# module building`
		`parser_build_module = subparsers.add_parser("build-module", help="Builds a given module in Copr")`
		`+ parser_build_module.add_argument("copr", help="The copr repo to list the packages of. Can be just name of project or even in format owner/project.", nargs="?")`
		`parser_build_module_mmd_source = parser_build_module.add_mutually_exclusive_group()`
		`parser_build_module_mmd_source.add_argument("--url", help="SCM with modulemd file in yaml format")`
		`parser_build_module_mmd_source.add_argument("--yaml", help="Path to modulemd file in yaml format")`
		`- parser_build_module.add_argument("--token",`
		`- help="OIDC token for module build service")`
		`parser_build_module.set_defaults(func="action_build_module")`

		`- parser_make_module = subparsers.add_parser("make-module", help="Makes a module in Copr")`
		`- parser_make_module.add_argument("copr",`
		`- help="The copr repo to build the module in. Can be just name of project or even in format username/project or @groupname/project.")`
		`- parser_make_module.add_argument("--yaml",`
		`- help="Path to modulemd file in yaml format")`
		`- parser_make_module.set_defaults(func="action_make_module")`
		`-`
		`return parser`

cli/copr_cli/util.py

file modified

-57

		`@@ -1,8 +1,5 @@`
		`# coding: utf-8`

		`- import socket`
		`- from datetime import timedelta`
		`-`
		`try:`
		`from progress.bar import Bar`
		`except ImportError:`
		`@@ -53,57 +50,3 @@`
		`suffix = "%(downloaded)s %(download_speed)s eta %(eta_td)s"`
		`else:`
		`ProgressBar = DummyBar`
		`-`
		`-`
		`- def listen_for_token():`
		`- """`
		`- Function taken from https://pagure.io/fm-orchestrator/blob/master/f/contrib/submit_build.py`
		`- We should avoid code duplicity by including it into _some_ python module`
		`-`
		`- Listens on port 13747 on localhost for a redirect request by OIDC`
		`- server, parses the response and returns the "access_token" value.`
		`- """`
		`- TCP_IP = '0.0.0.0'`
		`- TCP_PORT = 13747`
		`- BUFFER_SIZE = 1024`
		`-`
		`- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)`
		`- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)`
		`- s.bind((TCP_IP, TCP_PORT))`
		`- s.listen(1)`
		`-`
		`- conn, addr = s.accept()`
		`- data = ""`
		`- sent_resp = False`
		`- while 1:`
		`- try:`
		`- r = conn.recv(BUFFER_SIZE)`
		`- except:`
		`- conn.close()`
		`- break`
		`- if not r: break`
		`- data += r.decode("utf-8")`
		`-`
		`- if not sent_resp:`
		`- response = "Token has been handled."`
		`- conn.send("""HTTP/1.1 200 OK`
		`- Content-Length: {}`
		`- Content-Type: text/plain`
		`- Connection: Closed`
		`-`
		`- {}""".format(len(response), response).encode("utf-8"))`
		`- conn.close()`
		`- sent_resp = True`
		`-`
		`- s.close()`
		`-`
		`- data = data.split("\n")`
		`- for line in data:`
		`- variables = line.split("&")`
		`- for var in variables:`
		`- kv = var.split("=")`
		`- if not len(kv) == 2:`
		`- continue`
		`- if kv[0] == "access_token":`
		`- return kv[1]`
		`- return None`

cli/man/copr-cli.1.asciidoc

file modified

+3 -18

		`@@ -87,9 +87,6 @@`
		`mock-config::`
		`Get the mock profile (similar to koji mock-config)`

		`- make-module::`
		`- Create module in Copr project (do not properly build it via MBS)`
		`-`
		`build-module::`
		`Build module via Copr MB`

		`@@ -608,34 +605,22 @@`
		`MODULE ACTIONS`
		`--------------`

		- `copr-cli make-module [options]`
		`- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
		`-`
		`- usage: copr make-module [-h] [--yaml YAML] copr`
		`-`
		`- Create module in Copr project (do not properly build it via MBS)`
		`-`
		`- --yaml YAML:`
		`- Path to modulemd file in yaml format`
		`-`
		`-`
		`copr-cli build-module [options]`
		`~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
		`- usage: copr build-module [-h] [--url URL] [--token TOKEN]`
		`+ usage: copr build-module [-h] [--url URL] [--token TOKEN] [copr]`

		`Build module via Copr MBS`

		`--url URL:`
		`SCM with modulemd file in yaml format`

		`- --token TOKEN:`
		`- OIDC token for module build service`
		`+ --yaml YAML:`
		`+ Path to modulemd file in yaml format`


		`EXAMPLES`
		`--------`

		`- copr-cli make-module --yaml testmodule.yaml ownername/projectname`
		`copr-cli build-module --url git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#620ec77`

frontend/coprs_frontend/coprs/config.py

file modified

		`@@ -43,6 +43,7 @@`

		`DIST_GIT_URL = None`
		`COPR_DIST_GIT_LOGS_URL = None`
		`+ MBS_URL = "http://copr-fe-dev.cloud.fedoraproject.org/module/1/module-builds/"`

		`# primary log file`
		`LOG_FILENAME = "/var/log/copr-frontend/frontend.log"`

frontend/coprs_frontend/coprs/filters.py

file modified

+1 -1

		`@@ -45,7 +45,7 @@`
		`def module_state_from_num(num):`
		`if num is None:`
		`return "unknown"`
		`- return ["pending", "succeeded", "failed"][num]`
		`+ return helpers.ModuleStatusEnum(num)`


		`@app.template_filter("os_name_short")`

frontend/coprs_frontend/coprs/forms.py

file modified

+11

		`@@ -585,6 +585,17 @@`
		`# @TODO Validate modulemd.yaml file`
		`])`

		`+ create = wtforms.BooleanField("create", default=True)`
		`+ build = wtforms.BooleanField("build", default=True)`
		`+`
		`+`
		`+ class ModuleBuildForm(wtf.Form):`
		`+ modulemd = FileField("modulemd")`
		`+ scmurl = wtforms.StringField()`
		`+ branch = wtforms.StringField()`
		`+ copr_owner = wtforms.StringField()`
		`+ copr_project = wtforms.StringField()`
		`+`

		`class ChrootForm(wtf.Form):`

frontend/coprs_frontend/coprs/helpers.py

file modified

		`@@ -101,6 +101,10 @@`
		`}`


		`+ class ModuleStatusEnum(with_metaclass(EnumType, object)):`
		`+ vals = {"pending": 0, "succeeded": 1, "failed": 2}`
		`+`
		`+`
		`class BuildSourceEnum(with_metaclass(EnumType, object)):`
		`vals = {"unset": 0,`
		`"srpm_link": 1, # url`

frontend/coprs_frontend/coprs/logic/modules_logic.py

file modified

+13 -1

		`@@ -5,6 +5,7 @@`
		`from coprs import models`
		`from coprs import db`
		`from coprs import exceptions`
		`+ from wtforms import ValidationError`


		`class ModulesLogic(object):`
		`@@ -32,13 +33,24 @@`
		`return cls.get_multiple().filter(models.Module.copr_id == copr.id)`

		`@classmethod`
		`- def from_modulemd(cls, yaml):`
		`+ def yaml2modulemd(cls, yaml):`
		`mmd = modulemd.ModuleMetadata()`
		`mmd.loads(yaml)`
		`+ return mmd`
		`+`
		`+ @classmethod`
		`+ def from_modulemd(cls, yaml):`
		`+ mmd = cls.yaml2modulemd(yaml)`
		`return models.Module(name=mmd.name, stream=mmd.stream, version=mmd.version, summary=mmd.summary,`
		`description=mmd.description, yaml_b64=base64.b64encode(yaml))`

		`@classmethod`
		`+ def validate(cls, yaml):`
		`+ mmd = cls.yaml2modulemd(yaml)`
		`+ if not all([mmd.name, mmd.stream, mmd.version]):`
		`+ raise ValidationError("Module should contain name, stream and version")`
		`+`
		`+ @classmethod`
		`def add(cls, user, copr, module):`
		`if not user.can_build_in(copr):`
		`raise exceptions.InsufficientRightsException("You don't have permissions to build in this copr.")`

frontend/coprs_frontend/coprs/models.py

file modified

		`@@ -1178,6 +1178,15 @@`
		`def action(self):`
		`return Action.query.filter(Action.object_type == "module").filter(Action.object_id == self.id).first()`

		`+ @property`
		`+ def state(self):`
		`+ """`
		`+ Return text representation of status of this build`
		`+ """`
		`+ if self.action is not None:`
		`+ return helpers.ModuleStatusEnum(self.action.result)`
		`+ return "-"`
		`+`
		`def repo_url(self, arch):`
		`# @TODO Use custom chroot instead of fedora-24`
		`# @TODO Get rid of OS name from module path, see how koji does it`

frontend/coprs_frontend/coprs/templates/coprs/detail/module.html

file modified

+10

		`@@ -118,6 +118,15 @@`
		`<h3 class="panel-title">How to use</h3>`
		`</div>`
		`<div class="panel-body">`
		`+ {% if module.state != 'succeeded' %}`
		`+ <div class="well well-lg">`
		`+ {% if module.state == 'failed' %}`
		`+ The module failed and therefore it cannot be enabled`
		`+ {% else %}`
		`+ The module is not built yet and therefore it cannot be enabled`
		`+ {% endif %}`
		`+ </div>`
		`+ {% else %}`
		`<p>To start working with modules, you first need to get modularity-enabled packages from <code>@copr/copr</code>: </p>`
		`<div class="highlight autumn">`
		<pre>dnf install dnf-plugins-core <span class="c1"># provides basic `dnf copr` command to enable COPR repos</span>
		`@@ -137,6 +146,7 @@`
		`dnf module <span class="nb">enable</span> {{ module.name }}</pre>`

		`</div>`
		`+ {% endif %}`
		`</div>`
		`</div>`
		`</div>`

frontend/coprs_frontend/coprs/views/api_ns/api_general.py

file modified

+57 -6

		`@@ -5,6 +5,9 @@`
		`import os`
		`import flask`
		`import sqlalchemy`
		`+ import json`
		`+ import requests`
		`+ from wtforms import ValidationError`

		`from werkzeug import secure_filename`

		`@@ -907,10 +910,43 @@`
		`})`


		`- @api_ns.route("/coprs/<username>/<coprname>/module/build/", methods=["POST"])`
		`+ @api_ns.route("/module/build/", methods=["POST"])`
		`+ @api_login_required`
		`+ def copr_build_module():`
		`+ form = forms.ModuleBuildForm(csrf_enabled=False)`
		`+ if not form.validate_on_submit():`
		`+ raise LegacyApiError(form.errors)`
		`+`
		`+ try:`
		`+ common = {"owner": flask.g.user.name,`
		`+ "copr_owner": form.copr_owner.data,`
		`+ "copr_project": form.copr_project.data}`
		`+ if form.scmurl.data:`
		`+ kwargs = {"json": dict({"scmurl": form.scmurl.data, "branch": form.branch.data}, **common)}`
		`+ else:`
		`+ kwargs = {"data": common, "files": {"yaml": form.modulemd.data}}`
		`+`
		`+ response = requests.post(flask.current_app.config["MBS_URL"], verify=False, **kwargs)`
		`+ if response.status_code == 500:`
		`+ raise LegacyApiError("Error from MBS: {} - {}".format(response.status_code, response.reason))`
		`+`
		`+ resp = json.loads(response.content)`
		`+ if response.status_code != 201:`
		`+ raise LegacyApiError("Error from MBS: {}".format(resp["message"]))`
		`+`
		`+ return flask.jsonify({`
		`+ "output": "ok",`
		`+ "message": "Created module {}-{}-{}".format(resp["name"], resp["stream"], resp["version"]),`
		`+ })`
		`+`
		`+ except requests.ConnectionError:`
		`+ raise LegacyApiError("Can't connect to MBS instance")`
		`+`
		`+`
		`+ @api_ns.route("/coprs/<username>/<coprname>/module/make/", methods=["POST"])`
		`@api_login_required`
		`@api_req_with_copr`
		`- def copr_build_module(copr):`
		`+ def copr_make_module(copr):`
		`form = forms.ModuleFormUploadFactory(csrf_enabled=False)`
		`if not form.validate_on_submit():`
		`# @TODO Prettier error`
		`@@ -919,20 +955,35 @@`
		`modulemd = form.modulemd.data.read()`
		`module = ModulesLogic.from_modulemd(modulemd)`
		`try:`
		`- module = ModulesLogic.add(flask.g.user, copr, module)`
		`- db.session.flush()`
		`- ActionsLogic.send_build_module(flask.g.user, copr, module)`
		`+ ModulesLogic.validate(modulemd)`
		`+ msg = "Nothing happened"`
		`+ if form.create.data:`
		`+ module = ModulesLogic.add(flask.g.user, copr, module)`
		`+ db.session.flush()`
		`+ msg = "Module was created"`
		`+`
		`+ if form.build.data:`
		`+ if not module.id:`
		`+ module = ModulesLogic.get_by_nsv(copr, module.name, module.stream, module.version).one()`
		`+ ActionsLogic.send_build_module(flask.g.user, copr, module)`
		`+ msg = "Module build was submitted"`
		`db.session.commit()`

		`return flask.jsonify({`
		`"output": "ok",`
		`- "message": "Module build was submitted",`
		`+ "message": msg,`
		`"modulemd": modulemd,`
		`})`

		`except sqlalchemy.exc.IntegrityError:`
		`raise LegacyApiError({"nsv": ["Module {} already exists".format(module.nsv)]})`

		`+ except sqlalchemy.orm.exc.NoResultFound:`
		`+ raise LegacyApiError({"nsv": ["Module {} doesn't exist. You need to create it first".format(module.nsv)]})`
		`+`
		`+ except ValidationError as ex:`
		`+ raise LegacyApiError({"nsv": [ex.message]})`
		`+`

		`@api_ns.route("/coprs/<username>/<coprname>/build-config/<chroot>/", methods=["GET"])`
		`@api_ns.route("/g/<group_name>/<coprname>/build-config/<chroot>/", methods=["GET"])`

python/copr/client/client.py

file modified

+21 -10

		`@@ -1466,24 +1466,35 @@`
		`response.handle = BaseHandle(client=self, response=response)`
		`return response`

		`- def build_module(self, modulemd, token):`
		`- endpoint = "/module/1/module-builds/"`
		`- url = urljoin(self.copr_url, endpoint)`
		`- headers = {"Authorization": "Bearer {}".format(token)}`
		`+ def build_module(self, modulemd, ownername=None, projectname=None):`
		`+ endpoint = "module/build"`
		`+ url = "{}/{}/".format(self.api_url, endpoint)`

		`+ data = {"copr_owner": ownername, "copr_project": projectname}`
		`if isinstance(modulemd, io.BufferedIOBase):`
		`- kwargs = {"files": {"yaml": modulemd}}`
		`+ data.update({"modulemd": (os.path.basename(modulemd.name), modulemd, "application/x-rpm")})`
		`else:`
		`- kwargs = {"json": {"scmurl": modulemd, "branch": "master"}}`
		`+ data.update({"scmurl": modulemd, "branch": "master"})`

		`- response = requests.post(url, headers=headers, **kwargs)`
		`+ def fetch(url, data, method):`
		`+ m = MultipartEncoder(data)`
		`+ monit = MultipartEncoderMonitor(m, lambda x: x)`
		`+ return self._fetch(url, monit, method="post", headers={'Content-Type': monit.content_type})`
		`+`
		`+ # @TODO Refactor process_package_action to be general purpose`
		`+ response = self.process_package_action(url, None, None, data=data, fetch_functor=fetch)`
		`return response`

		`- def make_module(self, projectname, modulemd, username=None):`
		`- api_endpoint = "module/build"`
		`+ def make_module(self, projectname, modulemd, username=None, create=True, build=True):`
		`+ api_endpoint = "module/make"`
		`ownername = username if username else self.username`
		`f = open(modulemd, "rb")`
		`- data = {"modulemd": (os.path.basename(f.name), f, "application/x-rpm"), "username": ownername}`
		`+ data = {`
		`+ "modulemd": (os.path.basename(f.name), f, "application/x-rpm"),`
		`+ "username": ownername,`
		`+ "create": "y" if create else "",`
		`+ "build": "y" if build else "",`
		`+ }`

		`url = "{0}/coprs/{1}/{2}/{3}/".format(`
		`self.api_url, ownername, projectname, api_endpoint`

frostyx commented 8 years ago

We need to get this PR merged before releasing any new version with modularity support

It allows frontend to act as gateway between copr-cli and MBS, so there is an auth only with copr API tokens
And some smaller, but important improvements which are clear from commit names
Also tests are supposed to work against this version