PR#6: add po consolidation for TM - fedora-l10n/fedora-localization-statistics

fedora-l10n / fedora-localization-statistics

#6 add po consolidation for TM

Merged 4 years ago by jibecfed. Opened 4 years ago by jibecfed.

f31-tm into master

add po consolidation for TM

Jean-Baptiste Holcroft • 4 years ago

e319bdc

build.py

file modified

+13 -8

		`@@ -11,7 +11,7 @@`
		`import tempfile`
		`import yaml`

		`- from shutil import copyfile`
		`+ from shutil import copyfile, copy2`
		`from translation_finder import discover`

		`def main():`
		`@@ -30,6 +30,7 @@`

		`srpm_folder="./src.rpms/f{v}/".format(v=args.release)`
		`result_folder="./results/f{v}/".format(v=args.release)`
		`+ tm_folder="./tm/f{v}/".format(v=args.release)`

		`pkgs = []`
		`for (dirpath, dirnames, filenames) in os.walk(srpm_folder):`
		`@@ -43,7 +44,7 @@`
		`package = [x for x in pkgs if x == args.srpm][0]`
		`srpm_file = "{srpm}/{a}".format(srpm=srpm_folder, a=package)`
		`extract_srpm(tmp, srpm_file, result_folder)`
		`- discover_translations(tmp, package, result_folder)`
		`+ discover_translations(tmp, package, result_folder, tm_folder)`
		`else:`
		`with tempfile.TemporaryDirectory() as tmp:`
		`if args.offset:`
		`@@ -60,7 +61,7 @@`

		`srpm_file = "{srpm}/{a}".format(srpm=srpm_folder, a=package)`
		`extract_srpm(tmp, srpm_file, result_folder)`
		`- discover_translations(tmp, package, result_folder)`
		`+ discover_translations(tmp, package, result_folder, tm_folder)`

		`subprocess.run(['./concat_csv.sh', result_folder],`
		`check=True)`
		`@@ -78,7 +79,7 @@`
		`out.close()`
		`error.close()`

		`- def discover_translations(tmp, name, result_folder):`
		`+ def discover_translations(tmp, name, result_folder, tm_folder):`
		`"""find po file"""`
		`print("discover_translations: "+tmp)`
		`translation_files = []`
		`@@ -102,7 +103,7 @@`
		`for translation in translation_files:`
		`# TODO: multiple translation files for same package gnome-clocks-3.32.0-1.fc30.src.rpm`
		`if translation["file_format"] == "po":`
		`- get_po_translation_level(tmp, translation, name, result_folder)`
		`+ get_po_translation_level(tmp, translation, name, result_folder, tm_folder)`
		`elif translation["file_format"] == "ts":`
		`get_ts_translation_level(tmp, translation, name, result_folder)`
		`elif translation["file_format"] == "json":`
		`@@ -113,7 +114,7 @@`
		`else:`
		`unknown_format(tmp, translation, name, translation["file_format"], result_folder)`

		`- def get_po_translation_level(path, mask, name, result_folder):`
		`+ def get_po_translation_level(path, mask, name, result_folder, tm_folder):`
		`filemask = mask["filemask"]`
		`print("get_po_translation_level: " + filemask)`

		`@@ -121,12 +122,16 @@`
		`error = open(result_folder + '/{p}.errors.txt'.format(p=name), 'a')`
		`subprocess.run(["pocount", filemask.split("*")[0], "--csv"],`
		`stdout=stats, stderr=error, check=True, cwd=path)`
		`- print(filemask.split("*")[0])`
		`- print(os.listdir(path))`

		`stats.close()`
		`error.close()`

		`+ # Copy translation files in translation memory`
		`+ for po in glob.glob(path +"/"+ filemask):`
		`+ dest = tm_folder +"/"+ name +"/"+ filemask.split("*")[0]`
		`+ os.makedirs(dest, exist_ok=True)`
		`+ copy2(po, dest)`
		`+`
		`subprocess.run(["sed", "-i", "-e", "s\|{p}\|.\|g".format(p=path),`
		`result_folder + '/{p}.errors.txt'.format(p=name)], check=True)`

build_tm.py

file added

+64

		`@@ -0,0 +1,64 @@`
		`+ #!/usr/bin/env python3`
		`+ """Consolidate each po files into compendium"""`
		`+`
		`+ import argparse`
		`+ import glob`
		`+ import os`
		`+ import subprocess`
		`+ import tempfile`
		`+`
		`+ def main():`
		`+ """Handle params"""`
		`+`
		`+ parser = argparse.ArgumentParser(`
		`+ description="Creates compendium for every languages")`
		`+`
		`+ parser.add_argument("--release", required=True, type=int, default=31,`
		`+ choices=[30, 31],`
		`+ help="Provide the Fedora release to analyze")`
		`+`
		`+ parser.add_argument("--lang", required=False, type=str,`
		`+ help="Filter a language to analyze")`
		`+`
		`+ args = parser.parse_args()`
		`+`
		`+ tm_folder="./tm/f{v}/".format(v=args.release)`
		`+`
		`+ pofiles = glob.glob(tm_folder+'//*.po')`
		`+ po_langs = list(set([ os.path.basename(po) for po in pofiles ]))`
		`+ po_langs.sort()`
		`+`
		`+ if len(args.lang) > 0:`
		`+ compute_lang(args.lang+".po", tm_folder)`
		`+ else:`
		`+ for langfile in po_langs:`
		`+ compute_lang(langfile, tm_folder)`
		`+`
		`+ def compute_lang(langfile, tm_folder):`
		`+ """ Generate compendium and convert it to tmx"""`
		`+ lang = os.path.splitext(langfile)[0]`
		`+ print("Creating lang: " + lang)`
		`+`
		`+ pofiles = glob.glob(tm_folder+'//'+langfile)`
		`+`
		`+ print(" - po consolidation")`
		`+ compendium_file = tm_folder + lang + ".po"`
		`+ command = ["pocompendium", compendium_file] + pofiles`
		`+ subprocess.run(command, check=True)`
		`+`
		`+ print(" - po to tmx convertion")`
		`+ tmx_file = tm_folder + lang + ".tmx"`
		`+ command = ["po2tmx", "--language="+lang, "--progress=none",`
		`+ compendium_file, "--output="+tmx_file]`
		`+ subprocess.run(command, check=True)`
		`+`
		`+ print(" - language terminology")`
		`+ terminology_file = tm_folder + lang + ".terminology.po"`
		`+ command = ["poterminology", "--ignore-case", "--fold-titlecase",`
		`+ "--inputs-needed", "1",`
		`+ "--progress=verbose", compendium_file, "--output="+terminology_file]`
		`+ subprocess.run(command, check=True)`
		`+`
		`+ if __name__ == '__main__':`
		`+ main()`
		`+`