- * Infrastructure is hosted by https://console-openshift-console.apps.ocp.fedoraproject.org/

-         srpm_regex = re.compile("^{}$".format(args.filter))

-     packages_folder = "./results/{v}/packages/".format(v=args.results)

-     srpms_path = os.path.abspath("./results/{v}/srpms/".format(v=args.results))

-     data_file = os.path.join("./results/{v}/".format(v=args.results), "data.json")

-     srpm_list_file = os.path.join(srpms_path, "{v}.txt".format(v=args.results))

-                 dnf_args = "-c {}".format(dnf_fp)

-                 log.info("Using dnf conf {}".format(dnf_file))

-                 log.warning("dnf conf {} not found".format(dnf_file))

-             "dnf {dnf_args} download --source --skip-broken --url '*' | grep src.rpm".format(dnf_args=dnf_args),

- """ Parse translation files to deduct language list """

- import csv

- from shutil import rmtree

-         description="Creates a list of languages form translation files"

-         "--describe", action="store_true", help="Describe the current list of languages"

-     )

- 

-     parser.add_argument(

-         "--analyzelang", type=str, help="Produce an analyze file for a language"

-     )

- 

-     parser.add_argument(

-         "--analyzealllangs",

-         action="store_true",

-         help="Produce an analyze file for all languages",

-     )

- 

-     parser.add_argument(

-     results_folder = "./results/{v}/".format(v=args.results)

-     lang_folder = os.path.join(results_folder, "languages/")

-     lang_analyze_folder = os.path.join(results_folder, "languages-analyses/")

-     if args.describe:

-         log.info("Describing detecting languages")

-         describe(lang_folder)

- 

-     elif args.analyzealllangs:

-         log.info("Provide more data to analyze errors")

-         rmtree(lang_analyze_folder, ignore_errors=True)

-         os.mkdir(lang_analyze_folder)

- 

-         langs = [

-             f

-             for f in os.listdir(lang_folder)

-             if os.path.isfile(os.path.join(lang_folder, f))

-         ]

-         for lang in sorted(langs):

-             analyze = analyze_lang(lang_folder, lang[: -len(".json")])

- 

-             with open(os.path.join(lang_analyze_folder, lang), "w") as f:

-                 f.write(json.dumps(analyze, indent=2))

- 

-     elif args.analyzelang:

-         log.info("Provide more data to analyze errors")

-         if not os.path.exists(lang_analyze_folder):

-             os.makedirs(lang_analyze_folder)

- 

-         analyze = analyze_lang(lang_folder, args.analyzelang)

-         result_file = os.path.join(lang_analyze_folder, args.analyzelang + ".json")

- 

-         with open(result_file, "w") as f:

-             f.write(json.dumps(analyze, indent=2))

- 

-     if args.refresh and os.path.isdir(lang_folder):

-         rmtree(lang_folder)

- 

-     if os.path.exists(lang_folder) is False:

-         log.info("Detecting the list of languages")

-         os.makedirs(lang_folder)

-         po_langs = detect_languages(package_folder, results_folder)

- 

-         for lang in po_langs.keys():

-             with open(os.path.join(lang_folder, str(lang) + ".json"), "w") as f:

-                 f.write(json.dumps(po_langs[lang], indent=2))

- def analyze_lang(lang_folder, analized_lang):

-     """ Analyze one lang """

-     log = logging.getLogger("buildLanguageList.analyze_lang")

-     files = []

-     results = dict()

-     with open(os.path.join(lang_folder, analized_lang + ".json"), "r") as read_file:

-         files = json.load(read_file)["po"]

- 

-     log.info(" Analysing language {l}, with {c} files".format(l=analized_lang, c=len(files)))

- 

-     for file in files:

-         metadata = dict()

-         try:

-             metadata = polib.pofile(file).metadata

-         except OSError:

-             # maybe a polib bug? to investigate before using it in TM

-             metadata["Language"] = "error-os"

-         except TypeError:

-             metadata["Language"] = "error-type"

-         except UnicodeDecodeError:

-             # encoding error, to investigate before using it in TM

-             metadata["Language"] = "error-unicode"

- 

-         if "Language" not in metadata.keys():

-             metadata["Language"] = "zzz_null"

-         elif metadata["Language"] == "":

-             metadata["Language"] = "zzz_empty"

- 

-         language = results.get(metadata.get("Language"), dict())

- 

-         count = language.get("Count", 0)

-         count += 1

-         language["Count"] = count

- 

-         lang_files = language.get("Files", [])

-         lang_files.append(file)

-         language["Files"] = sorted(lang_files)

- 

-         plurals = language.get("Plural-Forms", [])

-         plurals.append(metadata.get("Plural-Forms"))

-         plurals = list(set(plurals))

-         language["Plural-Forms"] = plurals

- 

-         teams = language.get("Language-Team", [])

-         teams.append(metadata.get("Language-Team"))

-         teams = list(set(teams))

-         language["Language-Team"] = teams

- 

-         results[metadata.get("Language")] = language

- 

-     return dict(sorted(results.items(), key=lambda item: item[0]))

- 

- 

- def describe(lang_folder):

-     """ Provide the number of files per language """

-     log = logging.getLogger("buildLanguageList.describe")

-     langs = [

-         f

-         for f in os.listdir(lang_folder)

-         if os.path.isfile(os.path.join(lang_folder, f))

-     ]

- 

-     for lang in sorted(langs):

-         with open(os.path.join(lang_folder, lang), "r") as read_file:

-             files = json.load(read_file)

- 

-         log.info(" {l}:{c}".format(l=lang[:-len('.json')], c=len(files)))

- 

- 

- def detect_languages(package_folder, results_folder):

-     """ For each po file, detect metadatas and deduct the language     """

-     """ Requires: a file hierarchy with po files                       """

-     """ Returns: a dictionary of lists, key=lang code, value=file list """

-     log_file = os.path.join(results_folder, "build_language_list.log")

-     debug_file = list()

-         log.info("{c}/{t} {p}".format(c=count, t=total, p=package))

-             alls = json.load(read_file)

-         to_process = [p for p in alls if p["file_format"] == "po"]

-         for pattern in to_process:

-             mask = os.path.join(package_folder, package, pattern["filemask"])

-             p = re.compile(mask.replace("*", "(.*)").replace("+", r"\+"))

-             for po in glob.glob(mask):

-                 result = p.search(po)

-                 lang_code = result.group(1)

-                     metadata = polib.pofile(po).metadata

-                 lang, decision = choose_lang(lang_code, metadata, error)

-                 debug_file.append([

-                     po,

-                     lang_code,

-                     metadata.get("Language", ""),

-                     error,

-                     lang,

-                     str(decision),

-                 ])

-                 lang_result = langs.get(lang, dict())

-                 po_results = lang_result.get("po", list())

-                 po_results.append(po)

-                 lang_result["po"] = po_results

-                 langs[lang] = lang_result

-     with open(log_file, "w") as file_object:

-         write_file_object = csv.writer(file_object)

-         write_file_object.writerows(debug_file)

- def choose_lang(filename, metadata, error):

-     """ From a po file and its medata, choose the most likely language code """

-     """ By priority: the Language medata """

-     """ Returns: a language code """

- """For each package, compute stats"""

- import glob

- import shutil

- import polib

- import logging

- import utils

-         description="Computes stats for each package with translations"

-     results_folder = "./results/{v}/".format(v=args.results)

-     packages_folder = "./results/{v}/packages/".format(v=args.results)

-     packages_stats_folder = "./results/{v}/packages-stats/".format(v=args.results)

-     languages_folder = "./results/{v}/languages/".format(v=args.results)

-     languages_stats_folder = "./results/{v}/languages-stats/".format(v=args.results)

- 

-     for folder in [

-         packages_stats_folder,

-         languages_stats_folder

-     ]:

-         if args.refresh and os.path.isdir(folder):

-             shutil.rmtree(folder)

-     distribution_stats = dict()

- 

-     if not os.path.exists(packages_stats_folder):

-         os.makedirs(packages_stats_folder)

-         log.info(" {c}/{t} - {p}".format(c=count, t=len(packages), p=package))

-         src_folder = os.path.join(packages_folder, package)

-         stats_file = os.path.join(packages_stats_folder, package + ".json")

-         if os.path.isfile(stats_file) is False:

-             with open(os.path.join(packages_folder, package, "discover.json"), "r") as f:

-                 discoveries = json.load(f)

-             results = dict()

-             for discover in discoveries:

-                 files = glob.glob(os.path.join(src_folder, discover["filemask"]))

-                 if discover["file_format"] == "po":

-                     results[discover["filemask"]] = get_po_translation_level(

-                         files, stats_file

-                     )

-             if len(results) == 0:

-                 log.warning("No translation file found?")

-             else:

-                 with open(stats_file, "w") as f:

-                     json.dump(results, f, indent=2)

-         else:

-             with open(stats_file, "r") as f:

-                 results = json.load(f)

-         distribution_stats = extract_release_stats(distribution_stats, results)

-     log.info("Computing language stats")

-     languages = [f for f in os.listdir(languages_folder)]

-     count = 0

-     languages_stats_folder = languages_stats_folder

-     if not os.path.exists(languages_stats_folder):

-         os.makedirs(languages_stats_folder)

-     for language in sorted(languages):

-         count += 1

-         lang = language[:-5]

-         log.info(" {c}/{t} - {l}".format(c=count, t=len(languages), l=lang))

-         with open(os.path.join(languages_folder, language), "r") as f:

-             discoveries = json.load(f)

-         stats_file = os.path.join(languages_stats_folder, lang + ".json")

-         if os.path.isfile(stats_file):

-             continue

-         files = discoveries.get("po", [])

-         if files:

-             with open(stats_file, "w") as f:

-                 json.dump(get_po_translation_level(files, stats_file), f, indent=2)

-     log.info("Storing distribution stats")

-     distribution_file = os.path.join(results_folder, "release.json")

-     with open(os.path.join(results_folder, "data.json"), "r") as f:

-         distribution_stats["total_release_packages"] = len(json.load(f))

-     total_packages_files = list()

-     for base, dirs, files in os.walk(packages_folder):

-         for file in files:

-             if file != "discover.json":

-                 total_packages_files.append(os.path.join(base, file))

-     distribution_stats["total_packages_files"] = len(total_packages_files)

-     distribution_stats["total_packages"] = len(packages)

-     distribution_stats["nb_files"] = len(list(set(distribution_stats["files"])))

-     packages_with_stats = [f for f in os.listdir(packages_stats_folder) if os.path.isfile(os.path.join(packages_stats_folder, f))]

-     distribution_stats["total_packages_with_stats"] = len(packages_with_stats)

-     distribution_stats["total_languages"] = len(languages)

-     with open(distribution_file, "w") as f:

-         json.dump(distribution_stats, f, indent=2)

-     log.info("Searching for bugs ;)")

-     used_files = list(set(distribution_stats["files"]))

-     if len(total_packages_files) != len(used_files):

-         log.debug("source:{s} used: {u}".format(s=len(total_packages_files), u=len(used_files)))

-         missing_files = [source for source in total_packages_files if not source in used_files]

-         missing_files_po = [file for file in missing_files if file.endswith(".po")]

-         if len(missing_files_po) > 0:

-             log.debug("Some po files are missing")

-             distribution_file = os.path.join(results_folder, "build_stats.missing_po_files.json")

-             with open(distribution_file, "w") as f:

-                 json.dump(missing_files_po, f, indent=2)

-     log.info("done")

- def get_po_translation_level(files, stats_file):

-     """ Compute results """

-     log = logging.getLogger("buildStats.get_po_translation_level")

-     stats = dict()

-     for file in files:

-         # remove non standard comments

-         # taken from: https://github.com/translate/translate/blob/master/tools/pocommentclean

-         command = ["sed", "-i", "/^#$/d;/^#[^\:\~,\.]/d", file]

-         subprocess.run(command, check=True, capture_output=True)

-         try:

-             stat = calcstats(file)

-         except Exception as e:

-             log.error(" {f} triggered an {t} exception: {e}".format(f=file, t=type(e).__name__, e=e))

-             continue

-         keys = [

-             "translatedsourcewords",

-             "fuzzysourcewords",

-             "untranslatedsourcewords",

-             "translated",

-             "fuzzy",

-             "untranslated",

-             "translatedtargetwords",

-         ]

-         results = dict()

-         for key in keys:

-             results[key] = stat.get(key, 0)

-         results["team"] = get_language_team(file)

-         stats[file] = results

-     return stats

- def get_language_team(file):

-     log = logging.getLogger("buildStats.get_language_team")

-     metadata = dict()

-     try:

-         metadata = polib.pofile(file).metadata

-     except OSError:

-         # maybe a polib bug? to investigate before using it in TM

-         metadata["Language"] = "error-os"

-     except UnicodeDecodeError:

-         # encoding error, to investigate before using it in TM

-         metadata["Language"] = "error-unicode"

-     except TypeError:

-         # TypeError: '>' not supported between instances of 'str' and 'int'

-         metadata["Language"] = "error-valuerror"

- 

-     team = "Unknown..."

-         team = metadata["Language-Team"]

-     except KeyError:

-         log.debug("The file {f} have no Language team? Here are the metadata: {m}".format(f=file, m=metadata))

-     return team

- 

- 

- def extract_release_stats(results, files_stats):

-     log = logging.getLogger("buildStats.extract_release_stats")

-     number_of_packages = results.get("nb_packages", 0)

-     number_of_packages += 1

-     files = results.get("files", list())

-     total_source_words = results.get("totalsourcewords", 0)

- 

-     for template in files_stats:

-         maxresult = 0

-         for file in files_stats[template]:

-             translated = files_stats[template][file]["translatedsourcewords"]

-             untranslated = files_stats[template][file]["untranslatedsourcewords"]

-             maxresult = max(maxresult, translated + untranslated)

-             files.append(file)

- 

-         total_source_words += maxresult

- 

-     results = {

-         "nb_packages": number_of_packages,

-         "files": files,

-         "totalsourcewords": total_source_words,

-     }

- """Consolidate each po files into compendium"""

-         description="Creates useful translator files for every languages"

-     results_folder = "./results/{v}/".format(v=args.results)

-     debug_folder = os.path.join(results_folder, "debug_folder/")

-     os.makedirs(debug_folder, exist_ok=True)

-     log.info("Building the translation memory for every languages")

-     langs = [