From 3ea1489b9c4a7bfdd0933c98bec70eb5bfd18a92 Mon Sep 17 00:00:00 2001
From: Rafael dos Santos <rdossant@redhat.com>
Date: Dec 07 2018 12:10:19 +0000
Subject: [PATCH 1/5] summarizer: ignore cached metadata when given a yaml


Signed-off-by: Rafael dos Santos <rdossant@redhat.com>

---

diff --git a/_fedmod/cli.py b/_fedmod/cli.py
index a70e90a..f76c060 100644
--- a/_fedmod/cli.py
+++ b/_fedmod/cli.py
@@ -196,8 +196,9 @@ def lint(modulemd, min_level):
 @click.argument("modules", metavar='MODULES', nargs=-1, required=False)
 @click.option("--add-file", "-f", metavar="FILE", multiple=True,
               type=click.Path(exists=True),
-              help="Additional modulemd files to check."
-                   " Can be given multiple times.")
+              help="Modulemd files to check. "
+                   "Causes cached metadata to be ignored. "
+                   "Can be given multiple times.")
 @click.option("--tree", "-t", is_flag=True, default=False,
               help="Print output as a tree")
 def summarize(modules, add_file, tree):
diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py
index cdad351..0332b5c 100644
--- a/_fedmod/modulemd_summarizer.py
+++ b/_fedmod/modulemd_summarizer.py
@@ -25,6 +25,7 @@ gi.require_version('Modulemd', '1.0')  # noqa: E402
 from gi.repository import Modulemd
 
 import smartcols
+from collections import defaultdict
 
 from . import _repodata
 
@@ -81,38 +82,35 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree):
     print('\nHint: [d]efault')
 
 
-def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps):
-    for yaml in yaml_files:
-        assert yaml.endswith('.yaml'), "Not a yaml file: {}".format(yaml)
-
-        mmd_index, failures = Modulemd.index_from_file(yaml)
-        assert len(failures) == 0, failures
-
-        for module_name, index in mmd_index.items():
-            for module in index.get_streams().values():
-                # local modulemd files might miss some info like context or
-                # version. So let's make sure nsvc is consistent
-                ctxt = module.get_context() or ''
-                nsvc = f'{module_name}:{module.get_stream()}:' \
-                       f'{module.get_version()}:{ctxt}'
-                plist = list(module.get_profiles().keys())
-                profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist))
-
-                for dep in module.get_dependencies():
-                    deplist = set(deps.get(nsvc, []))
-                    for m, s in dep.peek_requires().items():
-                        deplist.add(f"{m}:{','.join(s.get())}"
-                                    if len(s.get()) else m)
-                    deps[nsvc] = sorted(deplist)
-
-            defaults = index.get_defaults()
-            if not defaults:
-                continue
+def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps):
+    mmd_index, failures = Modulemd.index_from_file(yaml)
+    assert len(failures) == 0, failures
+
+    for module_name, index in mmd_index.items():
+        for module in index.get_streams().values():
+            # local modulemd files might miss some info like context or
+            # version. So let's make sure nsvc is consistent
+            ctxt = module.get_context() or ''
+            nsvc = f'{module_name}:{module.get_stream()}:' \
+                   f'{module.get_version()}:{ctxt}'
+            plist = list(module.get_profiles().keys())
+            profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist))
+
+            for dep in module.get_dependencies():
+                deplist = set(deps.get(nsvc, []))
+                for m, s in dep.peek_requires().items():
+                    deplist.add(f"{m}:{','.join(s.get())}"
+                                if len(s.get()) else m)
+                deps[nsvc] = sorted(deplist)
+
+        defaults = index.get_defaults()
+        if not defaults:
+            continue
 
-            # Local module metadata can overwrite metadata from repo
-            dstreams[module_name] = defaults.peek_default_stream()
-            for s, pset in defaults.peek_profile_defaults().items():
-                dprofiles[module_name][s] = pset.get()
+        # Local module metadata can overwrite metadata from repo
+        dstreams[module_name] = defaults.peek_default_stream()
+        for s, pset in defaults.peek_profile_defaults().items():
+            dprofiles[module_name][s] = pset.get()
 
 
 def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False):
@@ -122,16 +120,20 @@ def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False):
 
     *restrict_list*: if present, restricts output to modules supplied
 
-    *yaml_files*: additional yaml files to parse and include in the summary
+    *yaml_files*: yaml files to parse and include in the summary. If any,
+                  locally cached metadata will be ignored
     """
 
-    profiles = _repodata.get_modules_profiles_lookup().copy()
-    dstreams = _repodata.get_modules_default_streams_lookup().copy()
-    dprofiles = _repodata.get_modules_default_profiles_lookup().copy()
-    deps = _repodata.get_modules_dependencies_lookup().copy()
-
+    profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {}
     if yaml_files:
-        _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps)
+        for yaml in yaml_files:
+            assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}"
+            _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps)
+    else:
+        profiles = _repodata.get_modules_profiles_lookup()
+        dstreams = _repodata.get_modules_default_streams_lookup()
+        dprofiles = _repodata.get_modules_default_profiles_lookup()
+        deps = _repodata.get_modules_dependencies_lookup()
 
     restrict_list = restrict_list or []
     _print_summary(profiles, dstreams, dprofiles, deps, restrict_list, as_tree)
diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py
index f3df520..4cd2a6b 100644
--- a/tests/test_module_summary.py
+++ b/tests/test_module_summary.py
@@ -54,8 +54,6 @@ class TestModuleSummary(object):
         summarize_modules(yaml_files=[spec_v2_yaml_path])
         out, err = capfd.readouterr()
 
-        assert self.matches('testmodule', 'master', '20180405123256',
-                            'c2c572ec', 'default', 'platform:f29', out)
         assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43',
                             'buildroot, container, default, minimal, ' +
                             'srpm-buildroot', 'compatible:v3,v4,extras,' +

From 629d25f6106a98899b9cf90a77b55a0d28991a08 Mon Sep 17 00:00:00 2001
From: Rafael dos Santos <rdossant@redhat.com>
Date: Dec 07 2018 12:12:14 +0000
Subject: [PATCH 2/5] summarizer: add glob pattern matching for filtering


This can be helpful if one wants to check all modules with a stream
called 'default': just pass '*:default:*' to summarize-module and the
ouptut will be restrict to those modules.

It's more helpful than just filtering by module name which can be easily
done with grep.

Signed-off-by: Rafael dos Santos <rdossant@redhat.com>

---

diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py
index 0332b5c..7f9c208 100644
--- a/_fedmod/modulemd_summarizer.py
+++ b/_fedmod/modulemd_summarizer.py
@@ -25,12 +25,13 @@ gi.require_version('Modulemd', '1.0')  # noqa: E402
 from gi.repository import Modulemd
 
 import smartcols
+from fnmatch import fnmatch
 from collections import defaultdict
 
 from . import _repodata
 
 
-def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree):
+def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree):
     tb = smartcols.Table()
     cl = tb.new_column('Name')
     cl.tree = as_tree
@@ -41,11 +42,11 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree):
     cl_deps = tb.new_column('Dependencies')
     parent_ln = {}
     for nsvc, plist in sorted(profiles.items()):
-        modname, sname, version, context = nsvc.split(':')
-
-        if restrict_to and modname not in restrict_to:
+        if mfilter and not any((fnmatch(nsvc, p) for p in mfilter)):
             continue
 
+        modname, sname, version, context = nsvc.split(':')
+
         def is_def_strm(s):
             return s == sdefaults.get(modname, '')
 
@@ -113,12 +114,13 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps):
             dprofiles[module_name][s] = pset.get()
 
 
-def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False):
+def summarize_modules(mfilter=None, yaml_files=None, as_tree=False):
     """
     Load Modulemd objects from each repository in repo_list and print a summary
     of the modules found with a summary of their streams and profiles.
 
-    *restrict_list*: if present, restricts output to modules supplied
+    *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern
+               can be used, e.g for filtering by stream: '*:master:*'
 
     *yaml_files*: yaml files to parse and include in the summary. If any,
                   locally cached metadata will be ignored
@@ -135,5 +137,5 @@ def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False):
         dprofiles = _repodata.get_modules_default_profiles_lookup()
         deps = _repodata.get_modules_dependencies_lookup()
 
-    restrict_list = restrict_list or []
-    _print_summary(profiles, dstreams, dprofiles, deps, restrict_list, as_tree)
+    mfilter = mfilter or []
+    _print_summary(profiles, dstreams, dprofiles, deps, mfilter, as_tree)
diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py
index 4cd2a6b..1a5628d 100644
--- a/tests/test_module_summary.py
+++ b/tests/test_module_summary.py
@@ -35,7 +35,7 @@ class TestModuleSummary(object):
                             'c2c572ec', 'default', 'platform:f29', out)
 
     def test_summarize_modules_restricted(self, capfd):
-        summarize_modules(['reviewboard', 'django'])
+        summarize_modules(['reviewboard*', 'django*'])
         out, err = capfd.readouterr()
 
         assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86',

From 8a38f68414d2834290107b3f95e0001e2828a6c9 Mon Sep 17 00:00:00 2001
From: Rafael dos Santos <rdossant@redhat.com>
Date: Dec 07 2018 12:13:12 +0000
Subject: [PATCH 3/5] summarizer: update method documentation


Signed-off-by: Rafael dos Santos <rdossant@redhat.com>

---

diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py
index 7f9c208..aa5d351 100644
--- a/_fedmod/modulemd_summarizer.py
+++ b/_fedmod/modulemd_summarizer.py
@@ -114,21 +114,25 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps):
             dprofiles[module_name][s] = pset.get()
 
 
-def summarize_modules(mfilter=None, yaml_files=None, as_tree=False):
+def summarize_modules(mfilter=None, yamls=None, as_tree=False):
     """
-    Load Modulemd objects from each repository in repo_list and print a summary
-    of the modules found with a summary of their streams and profiles.
+    Load Modulemd objects from each yaml file in the `yamls` list and print a
+    summary of the modules found with streams, context, version, profiles and
+    dependencies information.
+    If no files are passed, local cached metadata is used instead.
 
     *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern
                can be used, e.g for filtering by stream: '*:master:*'
 
-    *yaml_files*: yaml files to parse and include in the summary. If any,
-                  locally cached metadata will be ignored
+    *yamls*: list of yaml files to parse. If any, locally cached metadata will
+             be ignored
+
+    *as_tree*: print the summary in tree format, grouping information.
     """
 
     profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {}
-    if yaml_files:
-        for yaml in yaml_files:
+    if yamls:
+        for yaml in yamls:
             assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}"
             _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps)
     else:
diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py
index 1a5628d..470b786 100644
--- a/tests/test_module_summary.py
+++ b/tests/test_module_summary.py
@@ -51,7 +51,7 @@ class TestModuleSummary(object):
                                 'c2c572ec', 'default', 'platform:f29', out)
 
     def test_summarize_modules_local_files(self, capfd):
-        summarize_modules(yaml_files=[spec_v2_yaml_path])
+        summarize_modules(yamls=[spec_v2_yaml_path])
         out, err = capfd.readouterr()
 
         assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43',

From 2030d2be1d36509b33b3ad3f60cdc0295b01ae05 Mon Sep 17 00:00:00 2001
From: Rafael dos Santos <rdossant@redhat.com>
Date: Dec 07 2018 12:13:18 +0000
Subject: [PATCH 4/5] summarizer: read metadata from url in cmdline


It's useful to have a summary of modules from a repository given its url
without having to cache it first.

Signed-off-by: Rafael dos Santos <rdossant@redhat.com>

---

diff --git a/_fedmod/cli.py b/_fedmod/cli.py
index f76c060..aa76245 100644
--- a/_fedmod/cli.py
+++ b/_fedmod/cli.py
@@ -199,8 +199,12 @@ def lint(modulemd, min_level):
               help="Modulemd files to check. "
                    "Causes cached metadata to be ignored. "
                    "Can be given multiple times.")
+@click.option("--add-url", "-u", metavar="URL", multiple=True,
+              help="Repositories to read module metadata from."
+                   "Causes cached metadata to be ignored. "
+                   "Can be given multiple times.")
 @click.option("--tree", "-t", is_flag=True, default=False,
               help="Print output as a tree")
-def summarize(modules, add_file, tree):
+def summarize(modules, add_file, add_url, tree):
     """Prints a summary of available modules"""
-    summarize_modules(modules, add_file, tree)
+    summarize_modules(modules, add_file, add_url, tree)
diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py
index aa5d351..e7f7e0f 100644
--- a/_fedmod/modulemd_summarizer.py
+++ b/_fedmod/modulemd_summarizer.py
@@ -24,11 +24,12 @@ import gi
 gi.require_version('Modulemd', '1.0')  # noqa: E402
 from gi.repository import Modulemd
 
+import tempfile
 import smartcols
 from fnmatch import fnmatch
 from collections import defaultdict
 
-from . import _repodata
+from . import _repodata, _fetchrepodata
 
 
 def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree):
@@ -83,11 +84,9 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree):
     print('\nHint: [d]efault')
 
 
-def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps):
-    mmd_index, failures = Modulemd.index_from_file(yaml)
-    assert len(failures) == 0, failures
-
-    for module_name, index in mmd_index.items():
+def _parse_mmd(mmd_index, profiles, dstreams, dprofiles, deps):
+    for index in mmd_index:
+        module_name = index.get_name()
         for module in index.get_streams().values():
             # local modulemd files might miss some info like context or
             # version. So let's make sure nsvc is consistent
@@ -114,12 +113,12 @@ def _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps):
             dprofiles[module_name][s] = pset.get()
 
 
-def summarize_modules(mfilter=None, yamls=None, as_tree=False):
+def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False):
     """
-    Load Modulemd objects from each yaml file in the `yamls` list and print a
-    summary of the modules found with streams, context, version, profiles and
-    dependencies information.
-    If no files are passed, local cached metadata is used instead.
+    Load Modulemd objects from each yaml file in the `yamls` list or each url
+    in `urls` list and print a summary of the modules found with streams,
+    context, version, profiles and dependencies information.
+    If no files and no ulrs are passed, local cached metadata is used instead.
 
     *mfilter*: if present, restricts output to the nsvc supplied. Glob pattern
                can be used, e.g for filtering by stream: '*:master:*'
@@ -127,6 +126,9 @@ def summarize_modules(mfilter=None, yamls=None, as_tree=False):
     *yamls*: list of yaml files to parse. If any, locally cached metadata will
              be ignored
 
+    *urls*: list of repositories to read from. If any, locally cached metadata
+            will be ignored.
+
     *as_tree*: print the summary in tree format, grouping information.
     """
 
@@ -134,7 +136,16 @@ def summarize_modules(mfilter=None, yamls=None, as_tree=False):
     if yamls:
         for yaml in yamls:
             assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}"
-            _add_module_metadata(yaml, profiles, dstreams, dprofiles, deps)
+            index, failures = Modulemd.index_from_file(yaml)
+            assert len(failures) == 0, failures
+            _parse_mmd(index.values(), profiles, dstreams, dprofiles, deps)
+    elif urls:
+        for url in urls:
+            with tempfile.TemporaryDirectory() as local_path:
+                rp = _fetchrepodata.RepoPaths(url, local_path)
+                _fetchrepodata._download_metadata_files(rp)
+                indexes = _fetchrepodata._read_modules(rp)
+                _parse_mmd(indexes, profiles, dstreams, dprofiles, deps)
     else:
         profiles = _repodata.get_modules_profiles_lookup()
         dstreams = _repodata.get_modules_default_streams_lookup()

From a1fb7c2e5bbe0c68a70fd48f8a72ada27dd1bdcc Mon Sep 17 00:00:00 2001
From: Rafael dos Santos <rdossant@redhat.com>
Date: Dec 07 2018 12:25:20 +0000
Subject: [PATCH 5/5] summarizer: replace assert by exception


Signed-off-by: Rafael dos Santos <rdossant@redhat.com>

---

diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py
index e7f7e0f..4731d42 100644
--- a/_fedmod/modulemd_summarizer.py
+++ b/_fedmod/modulemd_summarizer.py
@@ -27,6 +27,7 @@ from gi.repository import Modulemd
 import tempfile
 import smartcols
 from fnmatch import fnmatch
+from click import ClickException
 from collections import defaultdict
 
 from . import _repodata, _fetchrepodata
@@ -135,9 +136,10 @@ def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False):
     profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {}
     if yamls:
         for yaml in yamls:
-            assert yaml.endswith('.yaml'), f"Not a yaml file: {yaml}"
             index, failures = Modulemd.index_from_file(yaml)
-            assert len(failures) == 0, failures
+            if len(failures) != 0:
+                msgs = "\n".join((str(f.get_gerror()) for f in failures))
+                raise ClickException(f"Could not read {yaml}: {msgs}")
             _parse_mmd(index.values(), profiles, dstreams, dprofiles, deps)
     elif urls:
         for url in urls:

