From 65863decab264bd7f080aca5fbfb019599c009df Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Nov 07 2017 04:18:38 +0000 Subject: waiverdb CLI for creating new waivers https://pagure.io/waiverdb/issue/82 --- diff --git a/MANIFEST.in b/MANIFEST.in index 1013d57..50e52f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,4 +10,5 @@ include fedmsg.d/*.py recursive-include systemd * recursive-include conf * exclude conf/settings.py +exclude conf/client.conf recursive-include docs *.py *.rst *.inv Makefile diff --git a/README.md b/README.md index 85c3623..b7c83e9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # WaiverDB -WaiverDB is a companion service to -[ResultsDB](https://pagure.io/taskotron/resultsdb), for recording waivers +WaiverDB is a companion service to +[ResultsDB](https://pagure.io/taskotron/resultsdb), for recording waivers against test results. ## Quick development setup @@ -48,3 +48,27 @@ You can view fedmsgs published when new waivers get created by doing:: $ fedmsg-relay --config-filename fedmsg.d/config.py & $ fedmsg-tail --config fedmsg.d/config.py --no-validate --really-pretty + +### WaiverDB CLI +WaiverDB has a command-line client interface for creating new waivers against test +results. A sample configuration is installed as ``/usr/share/doc/waiverdb/client.conf.example``. +Copy it to ``/etc/waiverdb/client.conf`` and edit it there. Or you can use ``--config-file`` +to specify one. +``` +Usage: waiverdb-cli [OPTIONS] + + Creates new waivers against test results. + + Examples: + + waiverdb-cli -r 123 -r 456 -p "fedora-26" -c "It's dead!" + +Options: + -C, --config-file PATH Specify a config file to use + -r, --result-id INTEGER Specify one or more results to be waived + -p, --product-version TEXT Specify one of PDC's product version + identifiers. + --waived / --no-waived Whether or not the result is waived + -c, --comment TEXT A comment explaining why the result is waived + -h, --help Show this message and exit. +``` diff --git a/conf/client.conf.example b/conf/client.conf.example new file mode 100644 index 0000000..f58bf4f --- /dev/null +++ b/conf/client.conf.example @@ -0,0 +1,10 @@ +[waiverdb] +# Specify OIDC or Kerberos for authentication +auth_method=OIDC +api_url=https://waiverdb-web-waiverdb.app.os.stg.fedoraproject.org/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ +oidc_client_id=waiverdb-authorizer +oidc_client_secret=notsecret +oidc_scopes= + openid + https://waiverdb.fedoraproject.org/oidc/create-waiver diff --git a/requirements.txt b/requirements.txt index d7c5c31..8dd7afb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # This is a list of pypi packages to be installed into virtualenv. Alternatively, -# you can install these as RPMs instead of pypi packages. +# you can install these as RPMs instead of pypi packages. fedmsg[consumers,commands] m2crypto @@ -19,3 +19,9 @@ stomp.py # Documentation requirements sphinx sphinxcontrib-httpdomain + +# CLI +click +configparser +openidc-client +requests-kerberos diff --git a/setup.py b/setup.py index c6856e3..4d71a76 100644 --- a/setup.py +++ b/setup.py @@ -36,4 +36,7 @@ setup(name='waiverdb', license='GPLv2+', packages=['waiverdb', 'waiverdb.models'], package_dir={'waiverdb': 'waiverdb'}, + entry_points={ + 'console_scripts': ['waiverdb-cli=waiverdb.cli:cli'], + }, ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..0f28716 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: GPL-2.0+ +import pytest +import json +from mock import Mock, patch +from click.testing import CliRunner +from waiverdb.cli import cli as waiverdb_cli + + +def test_misconfigured_auth_method(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +api_url=http://localhost:5004/api/v1.0 + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: The config option "auth_method" is required\n' + + +def test_misconfigured_api_url(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: The config option "api_url" is required\n' + + +def test_misconfigured_oidc_id_provider(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: The config option "oidc_id_provider" is required\n' + + +def test_misconfigured_oidc_client_id(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: The config option "oidc_client_id" is required\n' + + +def test_misconfigured_oidc_scopes(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ +oidc_client_id=waiverdb + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: The config option "oidc_scopes" is required\n' + + +def test_no_product_version(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ +oidc_client_id=waiverdb +oidc_scopes= + openid + """) + runner = CliRunner() + args = ['-C', p.strpath] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: Please specify product version\n' + + +def test_no_result_id(tmpdir): + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ +oidc_client_id=waiverdb +oidc_scopes= + openid + """) + runner = CliRunner() + args = ['-C', p.strpath, '-p', 'fedora-26'] + result = runner.invoke(waiverdb_cli, args) + assert result.exit_code == 1 + assert result.output == 'Error: Please specify one or more result ids to waive\n' + + +def test_oidc_auth_is_enabled(tmpdir): + # Skip if waiverdb is rebuilt for an environment where Kerberos authentication + # is used and python-openidc-client is not available. + pytest.importorskip('openidc_client') + with patch('openidc_client.OpenIDCClient.send_request') as mock_oidc_req: + mock_rv = Mock() + mock_rv.json.return_value = { + "comment": "It's dead!", + "id": 15, + "product_version": "Parrot", + "result_id": 123, + "timestamp": "2017-010-16T17:42:04.209638", + "username": "foo", + "waived": True + } + mock_oidc_req.return_value = mock_rv + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=OIDC +api_url=http://localhost:5004/api/v1.0 +oidc_id_provider=https://id.stg.fedoraproject.org/openidc/ +oidc_client_id=waiverdb +oidc_scopes= + openid + """) + runner = CliRunner() + args = ['-C', p.strpath, '-p', 'Parrot', '-r', 123, '-c', "It's dead!"] + result = runner.invoke(waiverdb_cli, args) + exp_json = { + 'result_id': 123, + 'waived': True, + 'product_version': 'Parrot', + 'comment': "It's dead!" + } + mock_oidc_req.assert_called_once_with( + url='http://localhost:5004/api/v1.0/waivers/', + data=json.dumps(exp_json), + scopes=['openid'], + timeout=60, + headers={'Content-Type': 'application/json'}) + assert result.exit_code == 0 + assert result.output == 'Created waiver 15 for result 123\n' + + +def test_kerberos_is_enabled(tmpdir): + # Skip if waiverdb is rebuilt for an environment where OIDC authentication + # is used and python-requests-kerberos is not available. + pytest.importorskip('requests_kerberos') + with patch('requests.request') as mock_request: + mock_rv = Mock() + mock_rv.json.return_value = { + "comment": "It's dead!", + "id": 15, + "product_version": "Parrot", + "result_id": 123, + "timestamp": "2017-010-16T17:42:04.209638", + "username": "foo", + "waived": True + } + mock_request.return_value = mock_rv + p = tmpdir.join('client.conf') + p.write(""" +[waiverdb] +auth_method=Kerberos +api_url=http://localhost:5004/api/v1.0 + """) + runner = CliRunner() + args = ['-C', p.strpath, '-p', 'Parrot', '-r', 123, '-c', "It's dead!"] + result = runner.invoke(waiverdb_cli, args) + mock_request.assert_called_once() + assert result.output == 'Created waiver 15 for result 123\n' diff --git a/waiverdb.spec b/waiverdb.spec index fb743b9..57649c5 100644 --- a/waiverdb.spec +++ b/waiverdb.spec @@ -28,6 +28,8 @@ BuildRequires: python2-systemd BuildRequires: python2-pytest BuildRequires: python2-mock BuildRequires: python2-flask-oidc +BuildRequires: python2-configparser +BuildRequires: python2-click BuildRequires: stomppy %else # EPEL7 uses python- naming BuildRequires: python-setuptools @@ -40,6 +42,8 @@ BuildRequires: systemd-python BuildRequires: pytest BuildRequires: python-mock BuildRequires: python-flask-oidc +BuildRequires: python-click +BuildRequires: python-configparser BuildRequires: stomppy %endif BuildRequires: fedmsg @@ -59,6 +63,8 @@ Requires: python2-kerberos Requires: python2-systemd Requires: python2-mock Requires: python2-flask-oidc +Requires: python2-click +Requires: python2-configparser Requires: stomppy %else Requires: python-flask @@ -69,6 +75,8 @@ Requires: python-kerberos Requires: systemd-python Requires: python-mock Requires: python-flask-oidc +Requires: python-click +Requires: python-configparser Requires: stomppy %endif Requires: fedmsg @@ -106,6 +114,7 @@ py.test tests/ %endif %{python2_sitelib}/%{name} %{python2_sitelib}/%{name}*.egg-info +%attr(755,root,root) %{_bindir}/waiverdb-cli %{_unitdir}/%{name}.service %{_unitdir}/%{name}.socket diff --git a/waiverdb/cli.py b/waiverdb/cli.py new file mode 100644 index 0000000..53518b2 --- /dev/null +++ b/waiverdb/cli.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: GPL-2.0+ +""" +CLI for creating new waivers against test results. +""" +import click +import requests +import json +import configparser + +requests_session = requests.Session() + + +def validate_config(config): + """ + Validates the configuration needed for WaiverDB + :return: None or ClickException + """ + # Verify that all necessary config options are set + config_error = ('The config option "{0}" is required') + if not config.has_option('waiverdb', 'auth_method'): + raise click.ClickException(config_error.format('auth_method')) + auth_method = config.get('waiverdb', 'auth_method') + if auth_method not in ['OIDC', 'Kerberos']: + raise click.ClickException('The WaiverDB authentication mechanism of ' + '"{0}" is not supported'.format(auth_method)) + required_configs = ['api_url'] + if auth_method == 'OIDC': + required_configs.append('oidc_id_provider') + required_configs.append('oidc_client_id') + required_configs.append('oidc_scopes') + for required_config in required_configs: + if not config.has_option('waiverdb', required_config): + raise click.ClickException(config_error.format(required_config)) + + +@click.command(context_settings={'help_option_names': ['-h', '--help']}) +@click.option('--config-file', '-C', default='/etc/waiverdb/client.conf', + type=click.Path(exists=True), + help='Specify a config file to use') +@click.option('--result-id', '-r', multiple=True, type=int, + help='Specify one or more results to be waived') +@click.option('--product-version', '-p', + help='Specify one of PDC\'s product version identifiers.') +@click.option('--waived/--no-waived', default=True, + help='Whether or not the result is waived') +@click.option('--comment', '-c', + help='A comment explaining why the result is waived') +def cli(comment, waived, product_version, result_id, config_file): + """ + Creates new waivers against test results. + + Examples: + + waiverdb-cli -r 123 -r 456 -p "fedora-26" -c "It's dead!" + + """ + config = configparser.SafeConfigParser() + config.read(config_file) + validate_config(config) + + if not product_version: + raise click.ClickException('Please specify product version') + # This name makes more sense for the values of result_id, where as --result-id + # makes more sense from the cli perspective. + result_ids = result_id + if not result_ids: + raise click.ClickException('Please specify one or more result ids to waive') + + auth_method = config.get('waiverdb', 'auth_method') + if auth_method == 'OIDC': + # Try to import this now so the user gets immediate feedback if + # it isn't installed + try: + import openidc_client # noqa: F401 + except ImportError: + raise click.ClickException('python-openidc-client needs to be installed') + # Get the auth token using the OpenID client. + oidc_client_secret = None + if config.has_option('waiverdb', 'oidc_client_secret'): + oidc_client_secret = config.get('waiverdb', 'oidc_client_secret') + oidc = openidc_client.OpenIDCClient( + 'waiverdb', + config.get('waiverdb', 'oidc_id_provider'), + {'Token': 'Token', 'Authorization': 'Authorization'}, + config.get('waiverdb', 'oidc_client_id'), + oidc_client_secret) + for result_id in result_ids: + data = { + 'result_id': result_id, + 'waived': waived, + 'product_version': product_version, + 'comment': comment + } + api_url = config.get('waiverdb', 'api_url') + scopes = config.get('waiverdb', 'oidc_scopes').strip().splitlines() + resp = oidc.send_request( + scopes=scopes, + url='{0}/waivers/'.format(api_url.rstrip('/')), + data=json.dumps(data), + headers={'Content-Type': 'application/json'}, + timeout=60) + if not resp.ok: + try: + error_msg = resp.json()['message'] + except (ValueError, KeyError): + error_msg = resp.text + raise click.ClickException( + 'Failed to create waiver for result {0}:\n{1}' + .format(result_id, error_msg)) + click.echo('Created waiver {0} for result {1}'.format( + resp.json()['id'], result_id)) + elif auth_method == 'Kerberos': + # Try to import this now so the user gets immediate feedback if + # it isn't installed + try: + import requests_kerberos # noqa: F401 + except ImportError: + raise click.ClickException('python-requests-kerberos needs to be installed') + auth = requests_kerberos.HTTPKerberosAuth(mutual_authentication=requests_kerberos.OPTIONAL) + for result_id in result_ids: + data = { + 'result_id': result_id, + 'waived': waived, + 'product_version': product_version, + 'comment': comment + } + api_url = config.get('waiverdb', 'api_url') + resp = requests.request('POST', '{0}/waivers/'.format(api_url.rstrip('/')), + data=json.dumps(data), auth=auth, + headers={'Content-Type': 'application/json'}, + timeout=60) + if resp.status_code == 401: + raise click.ClickException('WaiverDB authentication using Kerberos failed. ' + 'Make sure you have a valid Kerberos ticket.') + if not resp.ok: + try: + error_msg = resp.json()['message'] + except (ValueError, KeyError): + error_msg = resp.text + raise click.ClickException( + 'Failed to create waiver for result {0}:\n{1}' + .format(result_id, error_msg)) + click.echo('Created waiver {0} for result {1}'.format( + resp.json()['id'], result_id)) + + +if __name__ == '__main__': + cli() # pylint: disable=E1120