From 6570089bf920da749d6659adb0a321de1ac3bd8b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Aug 26 2023 13:15:54 +0000 Subject: merge language stats files together move from two json files and one log file per language to one single json doing that allow to open the file once to get language code and debug information also, the patterns may cover multiple time the same file, let's save time by not opening the same time multiple times this simplifies everything, which is good --- diff --git a/build_language_list.py b/build_language_list.py index 828d079..cfd73f8 100755 --- a/build_language_list.py +++ b/build_language_list.py @@ -2,7 +2,6 @@ """ Parse translation files to deduct language list """ import argparse -import csv import glob import json import os @@ -31,20 +30,6 @@ def main(): parser.add_argument("--refresh", action="store_true", help="Force refresh") parser.add_argument( - "--describe", action="store_true", help="Describe the current list of languages" - ) - - parser.add_argument( - "--analyzelang", type=str, help="Produce an analyze file for a language" - ) - - parser.add_argument( - "--analyzealllangs", - action="store_true", - help="Produce an analyze file for all languages", - ) - - parser.add_argument( "-v", "--verbose", default=False, @@ -61,38 +46,6 @@ def main(): results_folder = "./results/{v}/".format(v=args.results) lang_folder = os.path.join(results_folder, "languages/") package_folder = os.path.join(results_folder, "packages/") - lang_analyze_folder = os.path.join(results_folder, "languages-analyses/") - - if args.describe: - log.info("Describing detecting languages") - describe(lang_folder) - - elif args.analyzealllangs: - log.info("Provide more data to analyze errors") - rmtree(lang_analyze_folder, ignore_errors=True) - os.mkdir(lang_analyze_folder) - - langs = [ - f - for f in os.listdir(lang_folder) - if os.path.isfile(os.path.join(lang_folder, f)) - ] - for lang in sorted(langs): - analyze = analyze_lang(lang_folder, lang[: -len(".json")]) - - with open(os.path.join(lang_analyze_folder, lang), "w") as f: - f.write(json.dumps(analyze, indent=2)) - - elif args.analyzelang: - log.info("Provide more data to analyze errors") - if not os.path.exists(lang_analyze_folder): - os.makedirs(lang_analyze_folder) - - analyze = analyze_lang(lang_folder, args.analyzelang) - result_file = os.path.join(lang_analyze_folder, args.analyzelang + ".json") - - with open(result_file, "w") as f: - f.write(json.dumps(analyze, indent=2)) if args.refresh and os.path.isdir(lang_folder): rmtree(lang_folder) @@ -100,7 +53,7 @@ def main(): if os.path.exists(lang_folder) is False: log.info("Detecting the list of languages") os.makedirs(lang_folder) - po_langs = detect_languages(package_folder, results_folder) + po_langs = scan_packages(package_folder) for lang in po_langs.keys(): with open(os.path.join(lang_folder, str(lang) + ".json"), "w") as f: @@ -109,79 +62,12 @@ def main(): log.info("done") -def analyze_lang(lang_folder, analized_lang): - """ Analyze one lang """ - log = logging.getLogger("buildLanguageList.analyze_lang") - files = [] - results = dict() - with open(os.path.join(lang_folder, analized_lang + ".json"), "r") as read_file: - files = json.load(read_file)["po"] - - log.info(" Analysing language {l}, with {c} files".format(l=analized_lang, c=len(files))) - - for file in files: - metadata = dict() - try: - metadata = polib.pofile(file).metadata - except OSError: - # maybe a polib bug? to investigate before using it in TM - metadata["Language"] = "error-os" - except TypeError: - metadata["Language"] = "error-type" - except UnicodeDecodeError: - # encoding error, to investigate before using it in TM - metadata["Language"] = "error-unicode" - - if "Language" not in metadata.keys(): - metadata["Language"] = "zzz_null" - elif metadata["Language"] == "": - metadata["Language"] = "zzz_empty" - - language = results.get(metadata.get("Language"), dict()) - - count = language.get("Count", 0) - count += 1 - language["Count"] = count - - lang_files = language.get("Files", []) - lang_files.append(file) - language["Files"] = sorted(lang_files) - - plurals = language.get("Plural-Forms", []) - plurals.append(metadata.get("Plural-Forms")) - plurals = list(set(plurals)) - language["Plural-Forms"] = plurals - - teams = language.get("Language-Team", []) - teams.append(metadata.get("Language-Team")) - teams = list(set(teams)) - language["Language-Team"] = teams - - results[metadata.get("Language")] = language - - return dict(sorted(results.items(), key=lambda item: item[0])) - - -def describe(lang_folder): - """ Provide the number of files per language """ - log = logging.getLogger("buildLanguageList.describe") - langs = [ - f - for f in os.listdir(lang_folder) - if os.path.isfile(os.path.join(lang_folder, f)) - ] - - for lang in sorted(langs): - with open(os.path.join(lang_folder, lang), "r") as read_file: - files = json.load(read_file) - - log.info(" {l}:{c}".format(l=lang[:-len('.json')], c=len(files))) - - -def detect_languages(package_folder, results_folder): - """ For each po file, detect metadatas and deduct the language """ - """ Requires: a file hierarchy with po files """ - """ Returns: a dictionary of lists, key=lang code, value=file list """ +def scan_packages(package_folder: str): + """ For each po file, detect metadata and deduct the language + Requires: a file hierarchy with po files + :param package_folder: + :return: a dictionary of lists, key=lang code, value=file lis + """ log = logging.getLogger("buildLanguageList.detect_languages") langs = {} packages = [ @@ -190,25 +76,36 @@ def detect_languages(package_folder, results_folder): if os.path.isdir(os.path.join(package_folder, f)) ] - log_file = os.path.join(results_folder, "build_language_list.log") - debug_file = list() count = 0 + processed_files_count = 0 + processed_files_duplicates_count = 0 total = len(packages) for package in sorted(packages): count += 1 log.info("{c}/{t} {p}".format(c=count, t=total, p=package)) discovery_file = os.path.join(package_folder, package, "discover.json") + processed_files = list() with open(discovery_file, "r") as read_file: - alls = json.load(read_file) + discover_patterns = json.load(read_file) - to_process = [p for p in alls if p["file_format"] == "po"] + po_patterns = [p for p in discover_patterns if p["file_format"] == "po"] - for pattern in to_process: - mask = os.path.join(package_folder, package, pattern["filemask"]) - p = re.compile(mask.replace("*", "(.*)").replace("+", r"\+")) + for pattern in po_patterns: + filemask = os.path.join(package_folder, package, pattern["filemask"]) + p = re.compile(filemask.replace("*", "(.*)").replace("+", r"\+")) + + for po in glob.glob(filemask): + + if po in processed_files: + # there is no need to process the file it were processed already + log.debug(f"{po} were already processed") + processed_files_duplicates_count += 1 + continue + + processed_files.append(po) + processed_files_count += 1 - for po in glob.glob(mask): result = p.search(po) lang_code = result.group(1) metadata = dict() @@ -224,35 +121,38 @@ def detect_languages(package_folder, results_folder): # maybe a polib bug? to investigate before using it in TM error = "error-os" - lang, decision = choose_lang(lang_code, metadata, error) + lang, decision = choose_language_code_from_po(lang_code, metadata) - debug_file.append([ - po, - lang_code, - metadata.get("Language", ""), - error, - lang, - str(decision), - ]) + debug_file = {"file": po, + "lang_in_path": lang_code, + "metadata_lang": metadata.get("Language", ""), + "metadata_plurals": metadata.get("Plural-Forms", ""), + "metadata_language_team": metadata.get("Language-Team", ""), + "polib_error": error, + "lang_code_chosen": lang, + "lang_code_decision": str(decision) + } lang_result = langs.get(lang, dict()) + po_results = lang_result.get("po", list()) - po_results.append(po) + po_results.append(debug_file) lang_result["po"] = po_results langs[lang] = lang_result - with open(log_file, "w") as file_object: - write_file_object = csv.writer(file_object) - write_file_object.writerows(debug_file) + log.info(f"Done {processed_files_count} files were processed, we skipped {processed_files_duplicates_count} duplicates") return langs -def choose_lang(filename, metadata, error): - """ From a po file and its medata, choose the most likely language code """ - """ By priority: the Language medata """ - """ Returns: a language code """ +def choose_language_code_from_po(filename: str, metadata: dict[str]) -> tuple[str, int]: + """ From a po file and its metadata, choose the most likely language code + By priority: the Language metadata + :param filename: the po file + :param metadata: + :return: a language code + """ log = logging.getLogger("buildLanguageList.choose_lang") lang = "noresult" diff --git a/build_stats.py b/build_stats.py index 50a5bea..ce3645e 100755 --- a/build_stats.py +++ b/build_stats.py @@ -119,8 +119,9 @@ def main(): files = discoveries.get("po", []) if files: + po_files = [f["file"] for f in files] with open(stats_file, "w") as f: - json.dump(get_po_translation_level(files, stats_file), f, indent=2) + json.dump(get_po_translation_level(po_files, stats_file), f, indent=2) log.info("Storing distribution stats") diff --git a/check_dnf_files.sh b/check_dnf_files.sh index 916afe1..3da7c48 100755 --- a/check_dnf_files.sh +++ b/check_dnf_files.sh @@ -26,9 +26,6 @@ function call_sall { /src/build_language_list.py --results "$results" podman run -it --rm -v ./:/src:z -v "$WORK_DIR"/results:/src/results:z -v ./srpms:/srpms:z --tmpfs /tmp:size=4G -e DNF_CONF=/src/"$dnf_conf" fedlocstats:latest \ - /src/build_language_list.py --results "$results" --analyzealllang - - podman run -it --rm -v ./:/src:z -v "$WORK_DIR"/results:/src/results:z -v ./srpms:/srpms:z --tmpfs /tmp:size=4G -e DNF_CONF=/src/"$dnf_conf" fedlocstats:latest \ /src/build_tm.py --results "$results" --compress podman run -it --rm -v ./:/src:z -v "$WORK_DIR"/results:/src/results:z -v ./srpms:/srpms:z --tmpfs /tmp:size=4G -e DNF_CONF=/src/"$dnf_conf" fedlocstats:latest \ diff --git a/runall.sh b/runall.sh index 5951544..cd04eae 100755 --- a/runall.sh +++ b/runall.sh @@ -20,9 +20,6 @@ podman run -it --rm -v ./:/src:z -v ./results:/src/results:z -v ./srpms:/srpms:z # ~ 18 m ./build_language_list.py --results "$results" -# ~ 18 m -./build_language_list.py --results "$results" --analyzealllang - # Creates useful translator files for every languages # ~ 3 h 00 LANG=C ./build_tm.py --results "$results" --compress