From 61bebc5f990247ec56cc574bfa1205acabf914cb Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Feb 01 2021 20:38:28 +0000
Subject: [PATCH 1/3] use multithreads


---

diff --git a/build.py b/build.py
index 1f10d01..e1b00c1 100755
--- a/build.py
+++ b/build.py
@@ -16,6 +16,9 @@ import tempfile
 import time
 import datetime as dt
 import re
+import logging
+import threading
+import concurrent.futures
 
 from shutil import copyfile
 from translation_finder import discover
@@ -52,6 +55,7 @@ def main():
     )
     args = parser.parse_args()
 
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
     srpm_regex = None
     if args.filter:
         srpm_regex = re.compile("^{}$".format(args.filter))
@@ -70,113 +74,132 @@ def main():
         list_file_stats = os.stat(srpm_list_file)
         last_mod = dt.datetime.fromtimestamp(list_file_stats.st_mtime)
         if dt.datetime.now() - last_mod < dt.timedelta(hours=24):
+            logging.debug("loading srpm list")
             with open(srpm_list_file) as f:
                 url_list = f.readlines()
 
     if not url_list:
-        print("Fetching SRPMs url list")
+        logging.info("Fetching SRPMs url list")
         p = subprocess.Popen(
             "dnf download --source --skip-broken --url '*' | grep src.rpm",
             stdout=subprocess.PIPE,
-            shell=True,
-        )
+            shell=True)
 
         urls = str(p.stdout.read(), "utf-8")
+        logging.debug("saving srpm list")
         with open(srpm_list_file, "w") as f:
             f.write(urls)
         url_list = urls.splitlines()
 
     # Load processing data, if any
     try:
+        logging.debug("loading previous data")
         with open(data_file) as f:
             data = json.load(f)
+        logging.debug("previous data loaded")
     except BaseException:
         data = {}
 
     count = 0
     total_urls = len(url_list)
 
-    with tempfile.TemporaryDirectory(prefix="l10n-stats") as tmp:
-        for line in url_list:
-            count += 1
-            url = urlparse(line.strip())
-            if not url.scheme:
-                continue
-            srpm_filename = os.path.basename(url.path)
-            srpm_data = dnf.subject.Subject(srpm_filename)
-            package = srpm_data.get_nevra_possibilities(forms=1)[0]
-
-            if srpm_regex and not srpm_regex.match(package.name):
-                continue
-
-            if package.name in data and not args.force:
-                # Compare version
-                known_package = dnf.subject.Subject(
-                    data[package.name]["srpm"]
-                ).get_nevra_possibilities(forms=1)[0]
-                if (
-                    rpm.labelCompare(
-                        (package.epoch, package.version, package.release),
-                        (
-                            known_package.epoch,
-                            known_package.version,
-                            known_package.release,
-                        ),
-                    )
-                    <= 0
-                ):
-                    print(
-                        " {c}/{t} skipping already processed {n}".format(
-                            c=count, t=total_urls, n=package.name
-                        )
-                    )
-                    continue
-
-            print(
-                " {c}/{t} processing {n}".format(c=count, t=total_urls, n=package.name)
-            )
+    lock = threading.Lock()
+    max_conn = threading.Semaphore(3)
+    max_ext = threading.Semaphore(3)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+      for line in url_list:
+          count += 1
+          url = urlparse(line.strip())
+          if not url.scheme:
+              continue
+          executor.submit(process_srpm, url, srpms_path, packages_folder, srpm_regex, lock, data, data_file, force=args.force, keep=args.keep, max_conn=max_conn, max_ext=max_ext)
+
+          #thr = threading.Thread(target=process_srpm, args=(package.name,))
+
 
-            srpm_path = os.path.join(srpms_path, srpm_filename)
-            if not os.path.isfile(srpm_path):
-                if url.scheme == "rsync":
-                    dl = subprocess.run(
+
+def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, data={}, data_file=None, force=False, keep=False, max_conn=None, max_ext=None):
+    try:
+      srpm_filename = os.path.basename(url.path)
+      srpm_data = dnf.subject.Subject(srpm_filename)
+      package = srpm_data.get_nevra_possibilities(forms=1)[0]
+
+      if srpm_regex and not srpm_regex.match(package.name):
+          return
+
+      already_processed = False
+      with lock:
+        logging.debug("%s: taking lock", package.name)
+
+        if package.name in data and not force:
+            # Compare version
+            known_package = dnf.subject.Subject(
+                data[package.name]["srpm"]).get_nevra_possibilities(forms=1)[0]
+            if rpm.labelCompare(
+                (package.epoch,
+                 package.version,
+                 package.release),
+                (known_package.epoch,
+                 known_package.version,
+                 known_package.release)) <= 0:
+               already_processed = True
+        logging.debug("%s: releasing lock", package.name)
+
+      if already_processed:
+        logging.info("%s: Already processed, skipping", package.name)
+        return
+
+      logging.info("%s: Processing", package.name)
+      srpm_path = os.path.join(srpms_path, srpm_filename)
+      if not os.path.isfile(srpm_path):
+          with max_conn:
+            logging.info("%s: Downloading", package.name)
+            if url.scheme == "rsync":
+                dl = subprocess.run(
                         ["rsync", url.geturl(), srpms_path],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
-                        stderr=subprocess.STDOUT,
-                    )
-                else:
-                    dl = subprocess.run(
+                        stderr=subprocess.STDOUT)
+            else:
+                dl = subprocess.run(
                         ["curl", "-L", "--remote-name", url.geturl()],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
-                        cwd=srpms_path,
-                    )
-
-                if dl.returncode:
-                    print("error downloading srpm:")
-                    print(dl.stdout)
-                    continue
-
-            extract_srpm(tmp, srpm_path, packages_folder)
-            (tsearch, tcopy, results) = discover_translations(
-                tmp, package.name, packages_folder
-            )
-
-            if not args.keep:
-                os.unlink(srpm_path)
-
-            # save processed srpm name & version
-            data[package.name] = {
-                "srpm": srpm_filename,
-                "tsearch": tsearch,
-                "tcopy": tcopy,
-                "results": results,
-            }
-
-            with open(data_file, "w") as f:
-                json.dump(data, f, indent=2)
+                        cwd=srpms_path)
+
+            if dl.returncode:
+                logging.error("%s: error downloading srpm:", package.name)
+                logging.error(dl.stdout)
+                return
+
+      with tempfile.TemporaryDirectory(prefix="l10n-stats") as tmp:
+        with max_ext:
+          logging.info("%s: Extracting", package.name)
+          extract_srpm(tmp, srpm_path, packages_folder)
+        logging.info("%s: Discovering", package.name)
+        (tsearch, tcopy, results) = discover_translations(
+            tmp, package.name, packages_folder)
+
+      if not keep:
+          os.unlink(srpm_path)
+
+      with lock:
+        logging.debug("%s: taking lock", package.name)
+        # save processed srpm name & version
+        data[package.name] = {
+            "srpm": srpm_filename,
+            "tsearch": tsearch,
+            "tcopy": tcopy,
+            "results": results}
+
+        with open(data_file, "w") as f:
+            json.dump(data, f, indent=2)
+        logging.debug("%s: releasing lock", package.name)
+
+      logging.info("%s: Completed", package.name)
+    except Exception as e:
+      logging.exception(e)
 
 
 def extract_srpm(tmp, name, dest_folder):
diff --git a/docker/Dockerfile.33 b/docker/Dockerfile.33
index 1fae1cb..b2c8d10 100644
--- a/docker/Dockerfile.33
+++ b/docker/Dockerfile.33
@@ -9,4 +9,5 @@ RUN pip install charamel
 RUN pip install git+https://github.com/WeblateOrg/translation-finder.git
 
 VOLUME /src
+VOLUME /srpms
 WORKDIR /src

From 40987e6c3898b45280068cf938c248f0d7ac089c Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Feb 01 2021 20:38:37 +0000
Subject: [PATCH 2/3] add jinja2 requirement


---

diff --git a/requirements.txt b/requirements.txt
index 6f584b3..354fd5d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 pandas
+jinja2
 polib
 weblate-language-data
 langtable

From 0baa533da4eaeaf9f434db92a81055fc422b0655 Mon Sep 17 00:00:00 2001
From: Francois Andrieu <darknao@fedoraproject.org>
Date: Feb 01 2021 20:38:41 +0000
Subject: [PATCH 3/3] Use logging in build scripts


added -v flag for additional verbosity

---

diff --git a/build.py b/build.py
index e1b00c1..72201bd 100755
--- a/build.py
+++ b/build.py
@@ -53,9 +53,21 @@ def main():
         dest="force",
         help="Ignore past progression state",
     )
+    parser.add_argument(
+        "-v",
+        default=False,
+        action="store_true",
+        dest="verbose",
+        help="Add verbosity"
+    )
     args = parser.parse_args()
 
-    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
+    log = logging.getLogger("srpmParser")
+
     srpm_regex = None
     if args.filter:
         srpm_regex = re.compile("^{}$".format(args.filter))
@@ -74,29 +86,29 @@ def main():
         list_file_stats = os.stat(srpm_list_file)
         last_mod = dt.datetime.fromtimestamp(list_file_stats.st_mtime)
         if dt.datetime.now() - last_mod < dt.timedelta(hours=24):
-            logging.debug("loading srpm list")
+            log.debug("loading srpm list")
             with open(srpm_list_file) as f:
                 url_list = f.readlines()
 
     if not url_list:
-        logging.info("Fetching SRPMs url list")
+        log.info("Fetching SRPMs url list")
         p = subprocess.Popen(
             "dnf download --source --skip-broken --url '*' | grep src.rpm",
             stdout=subprocess.PIPE,
             shell=True)
 
         urls = str(p.stdout.read(), "utf-8")
-        logging.debug("saving srpm list")
+        log.debug("saving srpm list")
         with open(srpm_list_file, "w") as f:
             f.write(urls)
         url_list = urls.splitlines()
 
     # Load processing data, if any
     try:
-        logging.debug("loading previous data")
+        log.debug("loading previous data")
         with open(data_file) as f:
             data = json.load(f)
-        logging.debug("previous data loaded")
+        log.debug("previous data loaded")
     except BaseException:
         data = {}
 
@@ -119,6 +131,7 @@ def main():
 
 
 def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, data={}, data_file=None, force=False, keep=False, max_conn=None, max_ext=None):
+    log = logging.getLogger("srpmParser.process_srpm")
     try:
       srpm_filename = os.path.basename(url.path)
       srpm_data = dnf.subject.Subject(srpm_filename)
@@ -129,7 +142,7 @@ def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, d
 
       already_processed = False
       with lock:
-        logging.debug("%s: taking lock", package.name)
+        log.debug("%s: taking lock", package.name)
 
         if package.name in data and not force:
             # Compare version
@@ -143,17 +156,17 @@ def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, d
                  known_package.version,
                  known_package.release)) <= 0:
                already_processed = True
-        logging.debug("%s: releasing lock", package.name)
+        log.debug("%s: releasing lock", package.name)
 
       if already_processed:
-        logging.info("%s: Already processed, skipping", package.name)
+        log.info("%s: Already processed, skipping", package.name)
         return
 
-      logging.info("%s: Processing", package.name)
+      log.info("%s: Processing", package.name)
       srpm_path = os.path.join(srpms_path, srpm_filename)
       if not os.path.isfile(srpm_path):
           with max_conn:
-            logging.info("%s: Downloading", package.name)
+            log.info("%s: Downloading", package.name)
             if url.scheme == "rsync":
                 dl = subprocess.run(
                         ["rsync", url.geturl(), srpms_path],
@@ -169,15 +182,15 @@ def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, d
                         cwd=srpms_path)
 
             if dl.returncode:
-                logging.error("%s: error downloading srpm:", package.name)
-                logging.error(dl.stdout)
+                log.error("%s: error downloading srpm:", package.name)
+                log.error(dl.stdout)
                 return
 
       with tempfile.TemporaryDirectory(prefix="l10n-stats") as tmp:
         with max_ext:
-          logging.info("%s: Extracting", package.name)
+          log.info("%s: Extracting", package.name)
           extract_srpm(tmp, srpm_path, packages_folder)
-        logging.info("%s: Discovering", package.name)
+        log.info("%s: Discovering", package.name)
         (tsearch, tcopy, results) = discover_translations(
             tmp, package.name, packages_folder)
 
@@ -185,7 +198,7 @@ def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, d
           os.unlink(srpm_path)
 
       with lock:
-        logging.debug("%s: taking lock", package.name)
+        log.debug("%s: taking lock", package.name)
         # save processed srpm name & version
         data[package.name] = {
             "srpm": srpm_filename,
@@ -195,15 +208,16 @@ def process_srpm(url, srpms_path, packages_folder, srpm_regex=None, lock=None, d
 
         with open(data_file, "w") as f:
             json.dump(data, f, indent=2)
-        logging.debug("%s: releasing lock", package.name)
+        log.debug("%s: releasing lock", package.name)
 
-      logging.info("%s: Completed", package.name)
+      log.info("%s: Completed", package.name)
     except Exception as e:
-      logging.exception(e)
+      log.exception(e)
 
 
 def extract_srpm(tmp, name, dest_folder):
     """extract srpm page"""
+    log = logging.getLogger("srpmParser.extract_srpm")
 
     srpm_log_file = os.path.join(dest_folder, "..", "extract_srpm.log")
     srpm_error_file = os.path.join(dest_folder, "..", "extract_srpm_error.log")
@@ -217,6 +231,7 @@ def extract_srpm(tmp, name, dest_folder):
 
 def discover_translations(tmp, pkg_name, dest_folder):
     """find po file"""
+    log = logging.getLogger("srpmParser.discover_translations")
 
     translation_files = []
     tsearch = 0
@@ -254,6 +269,7 @@ def discover_translations(tmp, pkg_name, dest_folder):
 
 
 def copy_translations(tmp, translation, pkg_name, dest_folder):
+    log = logging.getLogger("srpmParser.copy_translations")
     filemask = translation["filemask"]
 
     if translation["file_format"] in ["po", "json", "json-nested"]:
diff --git a/build_language_list.py b/build_language_list.py
index 1b9bb80..be6c896 100755
--- a/build_language_list.py
+++ b/build_language_list.py
@@ -7,6 +7,7 @@ import json
 import os
 import polib
 import re
+import logging
 
 from shutil import rmtree
 from weblate_language_data import aliases, languages, language_codes, countries
@@ -41,7 +42,20 @@ def main():
         help="Produce an analyze file for all languages",
     )
 
+    parser.add_argument(
+        "-v",
+        default=False,
+        action="store_true",
+        dest="verbose",
+        help="Add verbosity"
+    )
+
     args = parser.parse_args()
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
+    log = logging.getLogger("buildLanguageList")
 
     results_folder = "./results/{v}/".format(v=args.results)
     lang_folder = os.path.join(results_folder, "languages/")
@@ -49,11 +63,11 @@ def main():
     lang_analyze_folder = os.path.join(results_folder, "languages-analyses/")
 
     if args.describe:
-        print("Describing detecting languages")
+        log.info("Describing detecting languages")
         describe(lang_folder)
 
     elif args.analyzealllangs:
-        print("Provide more data to analyze errors")
+        log.info("Provide more data to analyze errors")
         rmtree(lang_analyze_folder, ignore_errors=True)
         os.mkdir(lang_analyze_folder)
 
@@ -69,7 +83,7 @@ def main():
                 f.write(json.dumps(analyze, indent=2))
 
     elif args.analyzelang:
-        print("Provide more data to analyze errors")
+        log.info("Provide more data to analyze errors")
         if not os.path.exists(lang_analyze_folder):
             os.makedirs(lang_analyze_folder)
 
@@ -80,7 +94,7 @@ def main():
             f.write(json.dumps(analyze, indent=2))
 
     elif args.refresh:
-        print("Refreshing the list of languages")
+        log.info("Refreshing the list of languages")
         rmtree(lang_folder, ignore_errors=True)
         os.mkdir(lang_folder)
 
@@ -93,14 +107,13 @@ def main():
 
 def analyze_lang(lang_folder, analized_lang):
     """ Analyze one lang """
+    log = logging.getLogger("buildLanguageList.analyze_lang")
     files = []
     results = dict()
     with open(os.path.join(lang_folder, analized_lang + ".json"), "r") as read_file:
         files = json.load(read_file)["po"]
 
-    print(
-        " Analysing language {l}, with {c} files".format(l=analized_lang, c=len(files))
-    )
+    log.info(" Analysing language {l}, with {c} files".format(l=analized_lang, c=len(files)))
 
     for file in files:
         metadata = dict()
@@ -147,6 +160,7 @@ def analyze_lang(lang_folder, analized_lang):
 
 def describe(lang_folder):
     """ Provide the number of files per language """
+    log = logging.getLogger("buildLanguageList.describe")
     langs = [
         f
         for f in os.listdir(lang_folder)
@@ -157,13 +171,14 @@ def describe(lang_folder):
         with open(os.path.join(lang_folder, lang), "r") as read_file:
             files = json.load(read_file)
 
-        print(" {l}:{c}".format(l=lang[: -len(".json")], c=len(files)))
+        log.info(" {l}:{c}".format(l=lang[:-len('.json')], c=len(files)))
 
 
 def detect_languages(package_folder, results_folder):
     """ For each po file, detect metadatas and deduct the language     """
     """ Requires: a file hierarchy with po files                       """
     """ Returns: a dictionary of lists, key=lang code, value=file list """
+    log = logging.getLogger("buildLanguageList.detect_languages")
     langs = {}
     packages = [
         f
@@ -230,6 +245,7 @@ def choose_lang(filename, metadata, error):
     """ From a po file and its medata, choose the most likely language code """
     """ By priority: the Language medata """
     """ Returns: a language code """
+    log = logging.getLogger("buildLanguageList.choose_lang")
 
     lang = "noresult"
     decision = 0
diff --git a/build_map.py b/build_map.py
index e518635..a5bd416 100755
--- a/build_map.py
+++ b/build_map.py
@@ -6,6 +6,7 @@ import pandas
 import geopandas
 import matplotlib.pyplot as plt
 from mpl_toolkits.axes_grid1 import make_axes_locatable
+import logging
 
 RESULT_FOLDER = ""
 
@@ -36,8 +37,17 @@ def main():
         help="Include non official languages?",
     )
 
+    parser.add_argument("-v", default=False,
+                        action="store_true", dest="verbose",
+                        help="Add verbosity")
     args = parser.parse_args()
 
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
+    log = logging.getLogger("buildMap")
+
     RESULT_FOLDER = "./results/{r}/stats/".format(r=args.results)
 
     file = RESULT_FOLDER + "/3.result.csv"
@@ -46,6 +56,7 @@ def main():
 
 def parse(file, include_english, include_nonofficial):
     """Call all cleaning functions"""
+    log = logging.getLogger("buildMap.parse")
     # load statistics
     data = load_results(file)
     info(data, "CSV loaded")
@@ -87,6 +98,7 @@ def parse(file, include_english, include_nonofficial):
 
 def load_results(file):
     """ load previously computed statistics """
+    log = logging.getLogger("buildMap.load_results")
     data = pandas.read_csv(file)
     data.columns = [
         "src",
@@ -129,6 +141,7 @@ def load_results(file):
 
 def clean_lpopulation(row):
     """ converts every numbers as percentage """
+    log = logging.getLogger("buildMap.clean_lpopulation")
     val = row.LPopulation
 
     if "%" in row.LPopulation:
@@ -141,6 +154,7 @@ def clean_lpopulation(row):
 
 def load_cldr(include_english, include_nonofficial):
     """ Load population + language statistics per territory """
+    log = logging.getLogger("buildMap.load_cldr")
     cldr_data = pandas.read_csv(
         "CLDR-raw/country_language_population_raw.txt", sep="\t"
     )
@@ -220,6 +234,7 @@ def load_cldr(include_english, include_nonofficial):
 
 def get_per_lang_numbers(data):
     """ agregate statistics per language """
+    log = logging.getLogger("buildMap.get_per_lang_numbers")
 
     total_source_messages = data.groupby(["src", "dirname"])["totalMessage"].max().sum()
     total_source_words = (
@@ -255,6 +270,7 @@ def get_per_lang_numbers(data):
 
 
 def add_ISO3(data):
+    log = logging.getLogger("buildMap.add_ISO3")
     country_codes = pandas.read_csv("geo_data/country-codes.csv")
     country_codes["ISO3166-1-Alpha-2"] = country_codes["ISO3166-1-Alpha-2"].str.lower()
 
@@ -270,6 +286,7 @@ def add_ISO3(data):
 
 def get_maps(cldr_data, per_lang_numbers):
     """ merge each """
+    log = logging.getLogger("buildMap.get_maps")
     # language merge:
     maps = pandas.merge(
         cldr_data,
@@ -317,6 +334,7 @@ def get_maps(cldr_data, per_lang_numbers):
 
 def get_data_per_territory(data):
     """ group result per territory """
+    log = logging.getLogger("buildMap.get_data_per_territory")
 
     data["message_coverage"] = (
         data["language_population"] * data["message_progress_percentage"]
@@ -346,6 +364,7 @@ def get_data_per_territory(data):
 
 def add_geojson(miaw):
     """ merge result data with geojson data """
+    log = logging.getLogger("buildMap.add_geojson")
     world = geopandas.read_file("geo_data/countries.geojson")
 
     country_codes = pandas.read_csv("geo_data/country-codes.csv")
@@ -363,6 +382,7 @@ def add_geojson(miaw):
 
 def draw_map(world, column, title, filename, cmap):
     """ s """
+    log = logging.getLogger("buildMap.draw_map")
     fig, axes = plt.subplots(1, 1, figsize=(20, 12))
     divider = make_axes_locatable(axes)
 
@@ -379,7 +399,8 @@ def draw_map(world, column, title, filename, cmap):
 
 def info(dataset, step):
     """Print basic informations about current dataset"""
-    print(" * " + step + " → we now have " + str(len(dataset)) + " rows")
+    log = logging.getLogger("buildMap.info")
+    log.info(" * "+step+" → we now have "+str(len(dataset))+" rows")
 
 
 def store(dataset, name):
diff --git a/build_stats.py b/build_stats.py
index 27807f7..eff6b73 100755
--- a/build_stats.py
+++ b/build_stats.py
@@ -7,6 +7,7 @@ import json
 import os
 import shutil
 import polib
+import logging
 
 from translate.tools.pocount import calcstats
 
@@ -23,9 +24,21 @@ def main():
     parser.add_argument(
         "--refresh", action="store_true", help="Clear results before computing"
     )
+    parser.add_argument(
+        "-v",
+        default=False,
+        action="store_true",
+        dest="verbose",
+        help="Add verbosity")
 
     args = parser.parse_args()
 
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
+    log = logging.getLogger("buildStats")
+
     packages_folder = "./results/{v}/packages/".format(v=args.results)
     packages_stats_folder = "./results/{v}/packages-stats/".format(v=args.results)
     languages_folder = "./results/{v}/languages/".format(v=args.results)
@@ -42,7 +55,7 @@ def main():
         if args.refresh and os.path.isdir(folder):
             shutil.rmtree(folder)
 
-    print("Computing packages stats")
+    log.info("Computing packages stats")
     packages = [
         f
         for f in os.listdir(packages_folder)
@@ -56,7 +69,7 @@ def main():
 
     for package in sorted(packages):
         count += 1
-        print(" {c}/{t} - {p}".format(c=count, t=len(packages), p=package))
+        log.info(" {c}/{t} - {p}".format(c=count, t=len(packages), p=package))
         with open(os.path.join(packages_folder, package, "discover.json"), "r") as f:
             discoveries = json.load(f)
 
@@ -82,7 +95,7 @@ def main():
             with open(stats_file, "w") as f:
                 json.dump(results, f, indent=2)
 
-    print("Storing distribution stats")
+    log.info("Storing distribution stats")
     if not os.path.exists(distribution_stats_folder):
         os.makedirs(distribution_stats_folder)
 
@@ -90,7 +103,7 @@ def main():
     with open(stats_file, "w") as f:
         json.dump(distribution_stats, f, indent=2)
 
-    print("Computing language stats")
+    log.info("Computing language stats")
     languages = [f for f in os.listdir(languages_folder)]
     count = 0
 
@@ -102,7 +115,7 @@ def main():
         count += 1
         lang = language[:-5]
 
-        print(" {c}/{t} - {l}".format(c=count, t=len(languages), l=lang))
+        log.info(" {c}/{t} - {l}".format(c=count, t=len(languages), l=lang))
         with open(os.path.join(languages_folder, language), "r") as f:
             discoveries = json.load(f)
 
@@ -119,17 +132,14 @@ def main():
 
 def get_po_translation_level(files, stats_file):
     """ Compute results """
+    log = logging.getLogger("buildStats.get_po_translation_level")
     stats = dict()
 
     for file in files:
         try:
             stat = calcstats(file)
         except Exception as e:
-            print(
-                " {f} triggered an {t} exception: {e}".format(
-                    f=file, t=type(e).__name__, e=e
-                )
-            )
+            log.error(" {f} triggered an {t} exception: {e}".format(f=file, t=type(e).__name__, e=e))
             continue
 
         keys = [
@@ -153,6 +163,7 @@ def get_po_translation_level(files, stats_file):
 
 
 def get_language_team(file):
+    log = logging.getLogger("buildStats.get_language_team")
     metadata = dict()
     try:
         metadata = polib.pofile(file).metadata
@@ -167,11 +178,12 @@ def get_language_team(file):
     try:
         team = metadata["Language-Team"]
     except KeyError:
-        print("The file {f} have no Language team? Here are the metadata: {m}".format(f=file, m=metadata))
+        log.error("The file {f} have no Language team? Here are the metadata: {m}".format(f=file, m=metadata))
     return team
 
 
 def extract_release_stats(results, files_stats):
+    log = logging.getLogger("buildStats.extract_release_stats")
     number_of_packages = results.get("nb_packages", 0)
     number_of_packages += 1
     number_of_files = results.get("nb_files", 0)
diff --git a/build_tm.py b/build_tm.py
index b38504b..aa49eb8 100755
--- a/build_tm.py
+++ b/build_tm.py
@@ -8,6 +8,7 @@ import os
 import subprocess
 import shutil
 import tempfile
+import logging
 
 from io import BytesIO
 from translate.convert import po2tmx
@@ -30,8 +31,17 @@ def main():
 
     parser.add_argument("--compress", action="store_true", help="Compress output files")
 
+    parser.add_argument("-v", default=False,
+                        action="store_true", dest="verbose",
+                        help="Add verbosity")
     args = parser.parse_args()
 
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel)
+    log = logging.getLogger("buildTm")
+
     results_folder = "./results/{v}/".format(v=args.results)
     lang_path = os.path.join(results_folder, "languages/")
     tm_folder = os.path.join(results_folder, "languages-tm/")
@@ -44,7 +54,7 @@ def main():
     if not os.path.exists(tm_folder):
         os.makedirs(tm_folder)
 
-    print("Building the translation memory for every languages")
+    log.info("Building the translation memory for every languages")
 
     langs = [
         f for f in os.listdir(lang_path) if os.path.isfile(os.path.join(lang_path, f))
@@ -53,7 +63,7 @@ def main():
     for lang in sorted(langs):
         lang_code = lang[: -len(".json")]
 
-        print(" {l}".format(l=lang_code))
+        log.info(" {l}".format(l=lang_code))
 
         with open(os.path.join(lang_path, lang), "r") as read_file:
             files = json.load(read_file)["po"]
@@ -66,7 +76,7 @@ def main():
             try:
                 process_compendium(files, compendium_file)
             except Exception as e:
-                print(
+                log.error(
                     " Compendium generation triggered an {t} exception: {e}".format(
                         t=type(e).__name__, e=e
                     )
@@ -77,7 +87,7 @@ def main():
             try:
                 process_tmx(lang_code, compendium_file, tmx_file)
             except Exception as e:
-                print(
+                log.error(
                     " TMX generation triggered an {t} exception: {e}".format(
                         t=type(e).__name__, e=e
                     )
@@ -88,23 +98,24 @@ def main():
             try:
                 process_terminology(compendium_file, terminology_file)
             except Exception as e:
-                print(
+                log.error(
                     " Terminology generation triggered an {t} exception: {e}".format(
                         t=type(e).__name__, e=e
                     )
                 )
 
-    print("Detecting missing files")
+    log.info("Detecting missing files")
     for lang in sorted(langs):
         check_lang(lang[: -len(".json")], tm_folder)
 
     if args.compress:
-        print("Compressing files")
+        log.info("Compressing files")
         compress(tm_folder)
 
 
 def process_compendium(langfiles, dest):
     """ Generate a compendium (a concatenation of all po files) """
+    log = logging.getLogger("buildTm.process_compendium")
 
     pofiles = [
         os.path.join(os.path.dirname(os.path.abspath(__file__)), f) for f in langfiles
@@ -135,7 +146,7 @@ def process_compendium(langfiles, dest):
                     ]
                     subprocess.run(command, check=True, cwd=tmp, capture_output=True)
                 except subprocess.CalledProcessError as e:
-                    print("Error with msguniq {i}, error: {e}".format(i=i, e=e))
+                    log.error("Error with msguniq {i}, error: {e}".format(i=i, e=e))
 
             count += 1
 
@@ -151,7 +162,7 @@ def process_compendium(langfiles, dest):
         try:
             subprocess.run(command, check=True, cwd=tmp, capture_output=True)
         except subprocess.CalledProcessError:
-            print(" msgcat exception...")
+            log.error(" msgcat exception...")
 
 
 def process_tmx(lang, source, dest):
@@ -203,23 +214,25 @@ def process_terminology(source, dest):
 
 def check_lang(lang, tm_folder):
     """ Check if expected files were generated """
+    log = logging.getLogger("buildTm.check_lang")
 
     compendium_file = os.path.join(tm_folder, lang + ".po")
     tmx_file = os.path.join(tm_folder, lang + ".tmx")
     terminology_file = os.path.join(tm_folder, lang + ".terminology.po")
 
     if not os.path.isfile(compendium_file):
-        print(" {l}-compendium is missing".format(l=lang))
+        log.warning(" {l}-compendium is missing".format(l=lang))
 
     if not os.path.isfile(tmx_file):
-        print(" {l}-tmx is missing".format(l=lang))
+        log.warning(" {l}-tmx is missing".format(l=lang))
 
     if not os.path.isfile(terminology_file):
-        print(" {l}-terminology is missing".format(l=lang))
+        log.warning(" {l}-terminology is missing".format(l=lang))
 
 
 def compress(folder):
     """ Compress files uzing gzip """
+    log = logging.getLogger("buildTm.compress")
 
     files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
 
diff --git a/build_website.py b/build_website.py
index b340094..09fdd60 100755
--- a/build_website.py
+++ b/build_website.py
@@ -9,6 +9,7 @@ import langtable
 import os
 import pandas as pd
 import shutil
+import logging
 
 
 def main():
@@ -22,8 +23,17 @@ def main():
 
     parser.add_argument("--refresh", action="store_true", help="Force refresh of files")
 
+    parser.add_argument("-v", default=False,
+                        action="store_true", dest="verbose",
+                        help="Add verbosity")
     args = parser.parse_args()
 
+    loglevel = logging.INFO
+    if args.verbose:
+      loglevel = logging.DEBUG
+    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=loglevel, force=True)
+    log = logging.getLogger("buildWebsite")
+
     results_folder = "./results/{v}/".format(v=args.results)
     langs_log = os.path.join(results_folder, "build_language_list.log")
     langs_stats = os.path.join(results_folder, "languages-stats")
@@ -53,12 +63,12 @@ def main():
 
         os.makedirs(folder, exist_ok=True)
 
-    print("Get distribution stats")
+    log.info("Get distribution stats")
     distribution_stats = json.load(
         open(os.path.join(distribution_folder, "release.json"))
     )
 
-    print("Prepare json files for packages")
+    log.info("Prepare json files for packages")
     packages = [
         d
         for d in os.listdir(packages_stats)
@@ -82,7 +92,7 @@ def main():
             val.append({"name": name, "progress": lang["progress"], "translated": lang["translated"], "team": lang["team"]})
             packages_langs_results[lang["lang_code"]] = val
 
-    print("Prepare json files for languages")
+    log.info("Prepare json files for languages")
     languages = [
         f
         for f in os.listdir(langs_stats)
@@ -103,7 +113,7 @@ def main():
             store_json_file(results, dest_file)
 
     # generate static content for languages
-    print("Generate static content for languages")
+    log.info("Generate static content for languages")
     languages = [
         f
         for f in os.listdir(data_langs_folder)
@@ -121,7 +131,7 @@ def main():
 
         generate_static_pages_langs(args.results, code, content, dest_file)
 
-    print("Generate static content for packages")
+    log.info("Generate static content for packages")
     packages = [
         f
         for f in os.listdir(data_pkgs_folder)
@@ -139,7 +149,7 @@ def main():
 
         generate_static_pages_packages(args.results, code, content, dest_file)
 
-    print("Copy translation memories")
+    log.info("Copy translation memories")
     languages = [
         f for f in os.listdir(tm_folder) if os.path.isfile(os.path.join(tm_folder, f))
     ]
@@ -152,6 +162,7 @@ def main():
 
 def consolidate_language_stats(stats_file, distribution_stats):
     """ From a CSV file, return key indicators """
+    log = logging.getLogger("buildWebsite.consolidate_language_stats")
     results = dict()
     total_words_distrib = distribution_stats.get("totalsourcewords", 0)
 
@@ -198,6 +209,7 @@ def consolidate_language_stats(stats_file, distribution_stats):
 
 def consolidate_package_stats(stats_file, log_files):
     """ From a CSV file, return key indicators """
+    log = logging.getLogger("buildWebsite.consolidate_package_stats")
     results = dict()
 
     fieldnames = {
@@ -224,7 +236,7 @@ def consolidate_package_stats(stats_file, log_files):
 
         # sometimes, no file were found, which means no stats can be used
         if len(tmp_df) == 0:
-            print(" The template {t} for {f} is empty".format(t=template, f=stats_file))
+            log.warning(" The template {t} for {f} is empty".format(t=template, f=stats_file))
             continue
 
         tmp_df["totalsourcewords"] = (
@@ -239,7 +251,7 @@ def consolidate_package_stats(stats_file, log_files):
     if len(dfs) > 1:
         stats_df = pd.concat(dfs)
     elif len(dfs) == 0:
-        print("There is no stats for {f}".format(f=stats_file))
+        log.warning("There is no stats for {f}".format(f=stats_file))
         return results
     else:
         stats_df = dfs[0]
@@ -267,7 +279,7 @@ def consolidate_package_stats(stats_file, log_files):
         line["translated"] = line["translatedsourcewords"]
 
         if total_source_words == 0:
-            print(
+            log.info(
                 " File {f} for file has translatedsourcewords = 0 in line {l}".format(
                     f=stats_file, l=line
                 )
@@ -277,7 +289,7 @@ def consolidate_package_stats(stats_file, log_files):
         try:
             line["progress"] = round((int(line["translatedsourcewords"]) / total_source_words) * 100)
         except OverflowError:
-            print(
+            log.info(
                 " File {f} has Translated={t} and Source={tot}".format(
                     f=stats_file,
                     t=line["translatedsourcewords"],
@@ -298,6 +310,7 @@ def consolidate_package_stats(stats_file, log_files):
 
 
 def generate_static_pages_langs(results, code, content, dest_file):
+    log = logging.getLogger("buildWebsite.generate_static_pages_langs")
     data = content
     data["lang_name_en"] = langtable.language_name(
         languageId=code, languageIdQuery="en"
@@ -319,6 +332,7 @@ def generate_static_pages_langs(results, code, content, dest_file):
 
 
 def generate_static_pages_packages(results, code, content, dest_file):
+    log = logging.getLogger("buildWebsite.generate_static_pages_packages")
     data = content
     data["results"] = results
     data["package"] = code