From da88756813d479898b5ff9c9250cd1f6dcc57ab0 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Jan 25 2021 22:33:48 +0000 Subject: [PATCH 1/2] website: display package progress as a list goal: translators should be able to filter translation progress and access to upstream translation team. package pages will be more focused on internal analysis (list of files considered with detailed translations?). --- diff --git a/build_stats.py b/build_stats.py index 90cad4b..27807f7 100755 --- a/build_stats.py +++ b/build_stats.py @@ -6,7 +6,7 @@ import glob import json import os import shutil -import tempfile +import polib from translate.tools.pocount import calcstats @@ -145,11 +145,32 @@ def get_po_translation_level(files, stats_file): for key in keys: results[key] = stat.get(key, 0) + results["team"] = get_language_team(file) + stats[file] = results return stats +def get_language_team(file): + metadata = dict() + try: + metadata = polib.pofile(file).metadata + except UnicodeDecodeError: + # encoding error, to investigate before using it in TM + metadata["Language"] = "error-unicode" + except OSError: + # maybe a polib bug? to investigate before using it in TM + metadata["Language"] = "error-os" + + team = "Unknown..." + try: + team = metadata["Language-Team"] + except KeyError: + print("The file {f} have no Language team? Here are the metadata: {m}".format(f=file, m=metadata)) + return team + + def extract_release_stats(results, files_stats): number_of_packages = results.get("nb_packages", 0) number_of_packages += 1 diff --git a/build_website.py b/build_website.py index 02c6e02..7b38c25 100755 --- a/build_website.py +++ b/build_website.py @@ -77,24 +77,18 @@ def main(): results = consolidate_package_stats(file_stats, log_files) store_json_file(results, dest_file) - langs_results = ( - results.get("equalsormorethan80percent", []) - + results.get("between50and80percent", []) - + results.get("lessorequalto50percent", []) - ) - - for langs in langs_results: - val = packages_langs_results.get(langs["lang_code"], []) - val.append({"name": name, "progress": langs["progress"]}) - packages_langs_results[langs["lang_code"]] = val + for lang in results.get("stats", []): + val = packages_langs_results.get(lang["lang_code"], []) + val.append({"name": name, "progress": lang["progress"], "translated": lang["translated"], "team": lang["team"]}) + packages_langs_results[lang["lang_code"]] = val print("Prepare json files for languages") - langs = [ + languages = [ f for f in os.listdir(langs_stats) if os.path.isfile(os.path.join(langs_stats, f)) ] - for lang in sorted(langs): + for lang in sorted(languages): if lang.endswith(".json"): code = lang[: -len(".json")] dest_file = os.path.join(data_langs_folder, code + ".json") @@ -110,12 +104,12 @@ def main(): # generate static content for languages print("Generate static content for languages") - langs = [ + languages = [ f for f in os.listdir(data_langs_folder) if os.path.isfile(os.path.join(data_langs_folder, f)) ] - for lang in sorted(langs): + for lang in sorted(languages): code = lang[: -len(".json")] dest_file = os.path.join(static_langs_folder, code + ".md") @@ -146,10 +140,10 @@ def main(): generate_static_pages_packages(args.results, code, content, dest_file) print("Copy translation memories") - langs = [ + languages = [ f for f in os.listdir(tm_folder) if os.path.isfile(os.path.join(tm_folder, f)) ] - for lang in langs: + for lang in languages: if lang.endswith(".gz"): shutil.copyfile( os.path.join(tm_folder, lang), os.path.join(static_tm_folder, lang) @@ -170,6 +164,7 @@ def consolidate_language_stats(stats_file, distribution_stats): "fuzzy": "int", "untranslated": "int", "translatedtargetwords": "int", + "team": "str", "totalsourcewords": "int", } @@ -214,6 +209,7 @@ def consolidate_package_stats(stats_file, log_files): "fuzzy": "int", "untranslated": "int", "translatedtargetwords": "int", + "team": "str", "totalsourcewords": "int", } @@ -253,7 +249,7 @@ def consolidate_package_stats(stats_file, log_files): lambda x: x["_merge"] == "left_only" ] - temp = ( + temp_translated = ( stats_df_w_lang.groupby(["lang_code"]) .agg( { @@ -262,8 +258,10 @@ def consolidate_package_stats(stats_file, log_files): ) .reset_index() .droplevel(1, axis=1) - .to_dict(orient="records") ) + temp_teams = stats_df_w_lang.groupby("lang_code")["team"].apply(lambda x: ",".join(x)) + temp = pd.merge(temp_translated, temp_teams, how="inner", on="lang_code").to_dict(orient="records") + for line in temp: line["progress"] = 0 p = 0 @@ -285,27 +283,16 @@ def consolidate_package_stats(stats_file, log_files): tot=total_source_words, ) ) - + line["translated"] = line["translatedsourcewords"] line["progress"] = p - results["totalsourcewords"] = total_source_words - results["count_languages"] = len(pd.unique(stats_df_w_lang["lang_code"])) - + results["stats"] = list() for line in sorted(temp, key=lambda k: k["progress"], reverse=True): del line["translatedsourcewords"] - if line["progress"] <= 50: - hop = results.get("lessorequalto50percent", []) - hop.append(line) - results["lessorequalto50percent"] = hop - elif line["progress"] < 80: - hop = results.get("between50and80percent", []) - hop.append(line) - results["between50and80percent"] = hop - else: - hop = results.get("equalsormorethan80percent", []) - hop.append(line) - results["equalsormorethan80percent"] = hop + results["stats"].append(line) + results["totalsourcewords"] = total_source_words + results["count_languages"] = len(pd.unique(stats_df_w_lang["lang_code"])) results["no_languages"] = stats_df_no_lang["filename"].tolist() return results diff --git a/templates/language.md b/templates/language.md index a879b1d..9ac40af 100644 --- a/templates/language.md +++ b/templates/language.md @@ -21,6 +21,8 @@ Download: * {{ "{{%" }} link "/{{ results }}/{{ lang_code }}.tmx.gz" {{ "%}}" }}{{ lang_code }} translation memory{{ "{{%" }} /link {{ "%}}" }} see [tmx](https://en.wikipedia.org/wiki/Translation_Memory_eXchange) Packages: +| Name | Translated words | Progress | Language team | +|------------------------------------------------------------------------------------------------|--------------------------|------------------------|--------------------| {% for package in packages -%} -* [{{ package.name }}]({{ '{{' }}< ref "/{{ results }}/package/{{ package.name }}.md" >{{ '}}' }}) ({{ package.progress }}) +[{{ package.name }}]({{ '{{' }}< ref "/{{ results }}/package/{{ package.name }}.md" >{{ '}}' }}) | {{ package.translated }} | {{ package.progress }} | {{ package.team }} | {% endfor %} diff --git a/templates/package.md b/templates/package.md index 0dc57a6..0d905ef 100644 --- a/templates/package.md +++ b/templates/package.md @@ -2,35 +2,15 @@ title: "{{ package }}" date: {{ now }} --- -The package {{ package }} is transtlated into {{ count_languages }} languages in Fedora {{ results }}. +The package {{ package }} is translated into {{ count_languages }} languages in Fedora {{ results }}. -## Languages with ≥80% words translated -{% if equalsormorethan80percent %} -{% for stat in equalsormorethan80percent -%} -[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }}) -{% endfor %} -{% else %} -None -{% endif %} +This packages reprends {{ totalsourcewords }} source words to be translated in each language. -## Languages with >50% and <80% words translated -{% if between50and80percent %} -{% for stat in between50and80percent -%} -[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }}) +| Language | Translated words | Progress | +|-------------------------------------------------------------------------------------------------------|-----------------------|---------------------| +{% for stat in stats -%} +| [{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) | {{ stat.translated }} | {{ stat.progress }} | {% endfor %} -{% else %} -None -{% endif %} - - -## Languages with ≤50% words translated -{% if lessorequalto50percent %} -{% for stat in lessorequalto50percent -%} -[{{ stat.lang_code }}]({{ '{{' }}< ref "/{{ results }}/language/{{ stat.lang_code }}.md" >{{ '}}' }}) ({{ stat.progress }}) -{% endfor %} -{% else %} -None -{% endif %} ## Errors {% if no_languages %} From 66b22b099cd48870bb2cde17e165495b18f5f8dd Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: Feb 01 2021 20:10:49 +0000 Subject: [PATCH 2/2] ensure translated value always is set --- diff --git a/build_website.py b/build_website.py index 7b38c25..b340094 100755 --- a/build_website.py +++ b/build_website.py @@ -264,17 +264,18 @@ def consolidate_package_stats(stats_file, log_files): for line in temp: line["progress"] = 0 - p = 0 + line["translated"] = line["translatedsourcewords"] + if total_source_words == 0: print( " File {f} for file has translatedsourcewords = 0 in line {l}".format( f=stats_file, l=line ) ) - line["progress"] = p + line["progress"] = 0 continue try: - p = round((int(line["translatedsourcewords"]) / total_source_words) * 100) + line["progress"] = round((int(line["translatedsourcewords"]) / total_source_words) * 100) except OverflowError: print( " File {f} has Translated={t} and Source={tot}".format( @@ -283,8 +284,6 @@ def consolidate_package_stats(stats_file, log_files): tot=total_source_words, ) ) - line["translated"] = line["translatedsourcewords"] - line["progress"] = p results["stats"] = list() for line in sorted(temp, key=lambda k: k["progress"], reverse=True):