From 4bc3030a2c9a16cd45551fb18f98b3398ae67f7f Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 08:19:28 +0000 Subject: [PATCH 1/7] set githook globally --- diff --git a/check.sh b/check.sh index b203ef0..287445c 100755 --- a/check.sh +++ b/check.sh @@ -11,4 +11,4 @@ if [ -z "$username" ] || [ -z "$email" ] ; then fi # add pre-commit hooks -git config core.hooksPath .githooks +git config --global core.hooksPath .githooks From f6aa23125fdcb12bf719616264e495393121229c Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 08:21:41 +0000 Subject: [PATCH 2/7] always overwrite githook to keep it up to date & add some debug --- diff --git a/build.py b/build.py index f2c4232..c26096a 100755 --- a/build.py +++ b/build.py @@ -66,11 +66,9 @@ def main(): clone_translated_source() make_components_list() - if args.commit_l10n == "true": - commit_l10n_repos() + commit_l10n_repos(args.commit_l10n) - if args.commit_tsources == "true": - commit_translated_source() + commit_translated_source(args.commit_tsources) def check_git_config(): @@ -152,8 +150,7 @@ def clone_translated_source(): "--quiet" ], check=True) - -def commit_translated_source(): +def commit_translated_source(push='true'): """Commit translated sources""" print("Commit translated sources", flush=True) repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" @@ -170,23 +167,24 @@ def commit_translated_source(): r_commit = subprocess.run(["git", "commit", "-m", "[" + lang + "]automatic update of translated content", "--quiet"], cwd=repo_dir) - if r_commit.returncode == 0: - subprocess.run(["git", "push", "--quiet"], check=True, cwd=repo_dir) + + if r_commit.returncode == 0 and push == 'true': + subprocess.run(['git', 'push', "--quiet"], check=True, cwd=repo_dir) else: print(" [" + lang + "] No changes to commit", flush=True) -def commit_l10n_repos(): +def commit_l10n_repos(push='true'): """Commit and push changes of localization repositories""" print("Commit and push changes of localization repositories", flush=True) repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/l10n/" # commit each folders of the l10n path - [commit_l10n(r) for r in next(os.walk(repo_dir))[1]] + [commit_l10n(r, push=push) for r in next(os.walk(repo_dir))[1]] -def commit_l10n(repo_name): +def commit_l10n(repo_name, push='true'): """Commit and push changes of a localization repo""" print("* commit_l10n: " + repo_name, flush=True) @@ -196,12 +194,13 @@ def commit_l10n(repo_name): try: # add hooks dir_path = os.path.dirname(os.path.abspath(__file__)) - if not os.path.isdir(os.path.join(repo_dir, ".githooks")): - shutil.copytree(os.path.join(dir_path, ".githooks"), os.path.join(repo_dir, ".githooks")) + # overwrite existing hooks anyway + shutil.copytree(os.path.join(dir_path, '.githooks'), os.path.join(repo_dir, '.githooks'), dirs_exist_ok=True) - shutil.copy(os.path.join(dir_path, "check.sh"), os.path.join(repo_dir, "check.sh")) - subprocess.run(["./check.sh"], check=True, cwd=repo_dir) - os.remove(os.path.join(repo_dir, "check.sh")) + # Should not be needed anymore since everything is globally set + shutil.copy(os.path.join(dir_path, 'check.sh'), os.path.join(repo_dir, 'check.sh')) + subprocess.run(['./check.sh'], check=True, cwd=repo_dir) + os.remove(os.path.join(repo_dir, 'check.sh')) subprocess.run(["git", "add", "."], check=True, cwd=repo_dir) @@ -214,7 +213,10 @@ def commit_l10n(repo_name): cwd=repo_dir) if r_commit.returncode == 0: - subprocess.run(["git", "push", "--quiet"], check=True, cwd=repo_dir) + # List what exactly just got commited, post hook + subprocess.run(['git', 'show', '--stat', '--name-status'], cwd=repo_dir) + if push == 'true': + subprocess.run(['git', 'push', "--quiet"], check=True, cwd=repo_dir) else: print(" No changes to commit", flush=True) except subprocess.CalledProcessError as e: From be4530de3c5c1ee901ebec98a50972963c827c01 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 08:21:43 +0000 Subject: [PATCH 3/7] include master script to handle mail notification --- diff --git a/Dockerfile b/Dockerfile index e9c0418..aa5815f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,3 +9,4 @@ ENV HOME /workspace COPY . /workspace RUN chmod -R g+w /workspace WORKDIR /workspace +ENTRYPOINT ["python3", "entrypoint.py"] diff --git a/entrypoint.py b/entrypoint.py new file mode 100644 index 0000000..80e36b0 --- /dev/null +++ b/entrypoint.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import os +import sys +import smtplib +import configparser +import subprocess +from email.message import EmailMessage + + +def send_mail(content): + if os.path.isfile("config.ini"): + config = configparser.ConfigParser() + config.read("config.ini") + + mail = EmailMessage() + mail.add_header("To", config['mail']['recipient']) + mail.add_header("From", "noreply@fedoraproject.org") + mail.add_header("Subject", "doctranslation error log") + mail.set_payload(content) + try: + smtp = smtplib.SMTP(config['mail']['relayHost']) + smtp.send_message(mail) + finally: + smtp.quit() + print("mail sent") + else: + print("no mail config detected, ignoring") + +if __name__ == '__main__': + print("running: %s" % (" ".join(sys.argv)), flush=True) + run = subprocess.run(['./build.py', ] + sys.argv[1::], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print("done", flush=True) + if run.stdout: + print(run.stdout.decode('utf8'), flush=True) + if run.returncode: + send_mail(run.stdout.decode('utf8')) + From 242b019e91325a80190efae7275f6a2fc37334c8 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 08:22:16 +0000 Subject: [PATCH 4/7] convert most args to 'real' boolean and add --nopush for staging use --- diff --git a/build.py b/build.py index c26096a..f8916f8 100755 --- a/build.py +++ b/build.py @@ -25,12 +25,14 @@ def main(): parser = argparse.ArgumentParser( description="Calls `./src-to-pot.sh` and `./po-to-src.sh` for all source repos") - parser.add_argument("--clone_sources", choices=["true", "false"], required=True, - help="Should we clone/update sources or use local content?") - parser.add_argument("--commit_l10n", choices=["true", "false"], required=True, - help="Should we commit changes on l10n repos?") - parser.add_argument("--commit_tsources", choices=["true", "false"], required=True, - help="Should we commit translated sources?") + parser.add_argument("--clone_sources", action='store_true', + help="Clone/update sources") + parser.add_argument("--commit_l10n", action='store_true', + help="Commit changes on l10n repos") + parser.add_argument("--commit_tsources", action='store_true', + help="Commit translated sources") + parser.add_argument("--nopush", dest='push', action='store_false', + help="Don't push to remote repositories") parser.add_argument("--component", required=False, help="Antora component to convert, all if unset") parser.add_argument("--module", required=False, @@ -60,15 +62,17 @@ def main(): if args.clone_po4a: clone_po4a(args.clone_po4a_version) - if args.clone_sources == "true": + if args.clone_sources: clone_sources() clone_translated_source() make_components_list() - commit_l10n_repos(args.commit_l10n) + if args.commit_l10n: + commit_l10n_repos(push=args.push) - commit_translated_source(args.commit_tsources) + if args.commit_tsources: + commit_translated_source(push=args.push) def check_git_config(): @@ -150,7 +154,7 @@ def clone_translated_source(): "--quiet" ], check=True) -def commit_translated_source(push='true'): +def commit_translated_source(push=True): """Commit translated sources""" print("Commit translated sources", flush=True) repo_dir = os.path.dirname(os.path.abspath(__file__)) + "/translated-sources/" @@ -168,13 +172,13 @@ def commit_translated_source(push='true'): "-m", "[" + lang + "]automatic update of translated content", "--quiet"], cwd=repo_dir) - if r_commit.returncode == 0 and push == 'true': + if r_commit.returncode == 0 and push: subprocess.run(['git', 'push', "--quiet"], check=True, cwd=repo_dir) else: print(" [" + lang + "] No changes to commit", flush=True) -def commit_l10n_repos(push='true'): +def commit_l10n_repos(push=True): """Commit and push changes of localization repositories""" print("Commit and push changes of localization repositories", flush=True) @@ -184,7 +188,7 @@ def commit_l10n_repos(push='true'): [commit_l10n(r, push=push) for r in next(os.walk(repo_dir))[1]] -def commit_l10n(repo_name, push='true'): +def commit_l10n(repo_name, push=True): """Commit and push changes of a localization repo""" print("* commit_l10n: " + repo_name, flush=True) @@ -215,12 +219,12 @@ def commit_l10n(repo_name, push='true'): if r_commit.returncode == 0: # List what exactly just got commited, post hook subprocess.run(['git', 'show', '--stat', '--name-status'], cwd=repo_dir) - if push == 'true': + if push: subprocess.run(['git', 'push', "--quiet"], check=True, cwd=repo_dir) else: print(" No changes to commit", flush=True) except subprocess.CalledProcessError as e: - print("** ERROR" + e.output) + print("** ERROR" + (e.output if e.output else "")) def clone_sources(): From 9a106264b4ab643408d5ae1ad811683bbe1a9d70 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 09:34:45 +0000 Subject: [PATCH 5/7] write error to stderr and use returncode to trigger mail notification --- diff --git a/build.py b/build.py index f8916f8..3ab96b6 100755 --- a/build.py +++ b/build.py @@ -7,6 +7,7 @@ import urllib.request import shutil import subprocess import time +import sys from urllib.parse import urlparse @@ -374,6 +375,12 @@ if __name__ == "__main__": main() print("--- %s seconds ---" % (round(time.time() - start_time, 2)), flush=True) - print("eventual errors:", flush=True) with open("errors.txt", "r") as f: - print(f.read(), flush=True) + errors = f.read().strip() + + if errors: + print("errors:", flush=True, file=sys.stderr) + print(errors, file=sys.stderr) + sys.exit(1) + else: + print("no error") From d7e87211a9210bf9f09b05481b27668bf89f5db3 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 09:34:50 +0000 Subject: [PATCH 6/7] reformat mail subject --- diff --git a/entrypoint.py b/entrypoint.py index 80e36b0..106604b 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -16,7 +16,7 @@ def send_mail(content): mail = EmailMessage() mail.add_header("To", config['mail']['recipient']) mail.add_header("From", "noreply@fedoraproject.org") - mail.add_header("Subject", "doctranslation error log") + mail.add_header("Subject", "%s error log" % (config['mail']['subjectPrefix'],)) mail.set_payload(content) try: smtp = smtplib.SMTP(config['mail']['relayHost']) From d56c0182d50968cc6997ede21868cc3af9b9dc56 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Dec 16 2020 09:34:50 +0000 Subject: [PATCH 7/7] Preserve utf8 encoding in mail body --- diff --git a/entrypoint.py b/entrypoint.py index 106604b..b0a79f6 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -14,6 +14,7 @@ def send_mail(content): config.read("config.ini") mail = EmailMessage() + mail.add_header('Content-Type', 'text/plain; charset=utf-8') mail.add_header("To", config['mail']['recipient']) mail.add_header("From", "noreply@fedoraproject.org") mail.add_header("Subject", "%s error log" % (config['mail']['subjectPrefix'],)) @@ -34,5 +35,5 @@ if __name__ == '__main__': if run.stdout: print(run.stdout.decode('utf8'), flush=True) if run.returncode: - send_mail(run.stdout.decode('utf8')) + send_mail(run.stdout)