From dc521a2b2d795014327ef46c9e97d9820f66aadd Mon Sep 17 00:00:00 2001
From: Sergey Turivniy <turivniy@gmail.com>
Date: Jan 03 2018 17:18:24 +0000
Subject: [PATCH 1/2] Add statistic page. Old one stable end new one


---

diff --git a/statictic_page/new_page/.gitignore b/statictic_page/new_page/.gitignore
new file mode 100644
index 0000000..931d5b2
--- /dev/null
+++ b/statictic_page/new_page/.gitignore
@@ -0,0 +1,3 @@
+.up*
+*.pyc
+.idea/
diff --git a/statictic_page/new_page/README.md b/statictic_page/new_page/README.md
new file mode 100644
index 0000000..3d0ed66
--- /dev/null
+++ b/statictic_page/new_page/README.md
@@ -0,0 +1,46 @@
+#### Get packages statistic and generate file
+
+Navigate to folder with tests:
+```
+$ cd AtomicCi/statictic_page/new_page/
+```
+
+Create python virtualenv with python3
+and activate it:
+```
+$ virtualenv -p python3 envname
+$ . envname/bin/activate
+```
+
+Install requirenments:
+```
+$ pip install -r requirenments.txt
+```
+
+Edit file <code>config.py</code>, add <code>upstreamfirst_api_key</code> and <code>dist_git_api_key</code>. You can generate then on settings page:
+```
+upstreamfirst_api_key = ''
+dist_git_api_key = ''
+
+UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/'
+DIST_GIT_URL = 'https://src.fedoraproject.org/'
+
+FILE_UPLOAD_2_WIKI = 'upload_to_wiki.txt'
+FILE_PACKAGES = 'upstreamfirst_packages.txt'
+```
+Also you can change names of the files <code>FILE_UPLOAD_2_WIKI</code> and <code>FILE_PACKAGES</code>
+
+To run code just execute in a command line:
+```
+$ python runner.py
+```
+
+File <code>upstreamfirst_packages.txt</code> will be created.
+You can simply check progress by executing following command in another terminal:
+```
+$ tail -f upstreamfirst_packages.txt
+```
+
+After script will be finished, find <code>upload_to_wiki.txt</code>. Use data to upload it to the wiki.
+
+The result: https://fedoraproject.org/wiki/CI/Tests/new-stat#This_page_is_automatically_updated._Do_not_edit.
\ No newline at end of file
diff --git a/statictic_page/new_page/config.py b/statictic_page/new_page/config.py
new file mode 100644
index 0000000..5262a6c
--- /dev/null
+++ b/statictic_page/new_page/config.py
@@ -0,0 +1,8 @@
+upstreamfirst_api_key = ''
+dist_git_api_key = ''
+
+UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/'
+DIST_GIT_URL = 'https://src.fedoraproject.org/'
+
+FILE_UPLOAD_2_WIKI = 'upload_to_wiki.txt'
+FILE_PACKAGES = 'upstreamfirst_packages.txt'
\ No newline at end of file
diff --git a/statictic_page/new_page/generate_file.py b/statictic_page/new_page/generate_file.py
new file mode 100644
index 0000000..2749247
--- /dev/null
+++ b/statictic_page/new_page/generate_file.py
@@ -0,0 +1,196 @@
+from tools import get_list_of_packages_from_the_file
+from config import FILE_UPLOAD_2_WIKI
+from config import FILE_PACKAGES
+
+def generate_file_upload_on_wiki(input_file):
+    """Generate file that will be uploaded on the wiki page
+
+    Args:
+        input_file (file): file with packages
+
+    Returns:
+        (file): file that will be uploaded on the wiki page
+    """
+
+    packages = get_list_of_packages_from_the_file(input_file)
+    packages_statistic = get_packages_statistic(packages)
+
+    with open(FILE_UPLOAD_2_WIKI, 'w') as upload_file:
+
+        upload_file.write('== This page is automatically updated. Do not edit. ==\n')
+        upload_file.write('{| class="wikitable sortable"\n')
+        upload_file.write('! style="text-align:left;" scope="col" rowspan="2" | Package\n')
+        upload_file.write('! scope="col" colspan="6"| dist-git\n')
+        upload_file.write('! scope="col" colspan="4"| upstreamfirst\n')
+        upload_file.write('|-\n')
+        upload_file.write('! scope="col" | tests.yml\n')
+        upload_file.write('! scope="col" | classic\n')
+        upload_file.write('! scope="col" | container\n')
+        upload_file.write('! scope="col" | atomic\n')
+        upload_file.write('! scope="col" | missing\n')
+        upload_file.write('! scope="col" | pending PR\n')
+        upload_file.write('! scope="col" | tests.yml\n')
+        upload_file.write('! scope="col" | classic\n')
+        upload_file.write('! scope="col" | container\n')
+        upload_file.write('! scope="col" | atomic\n')
+        upload_file.write('|-\n')
+
+        # Statistics
+        upload_file.write('! style="text-align:left; font-weight: lighter;" scope="row" | Total known packages\n')
+        upload_file.write('| style="text-align:center;" scope="row" colspan="10" | {}\n'.format(
+            packages_statistic['total']))
+        upload_file.write('|-\n')
+        upload_file.write('! style="text-align:left; font-weight: lighter;" scope="row" | Packages in category\n')
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_yml']))
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['classic']))
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['container']))
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['test_tags']['atomic']))
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['missing']))
+        upload_file.write('| {}\n'.format(packages_statistic['dist-git']['pending']))
+        upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_yml']))
+        upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['classic']))
+        upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['container']))
+        upload_file.write('| {}\n'.format(packages_statistic['upstreamfirst']['test_tags']['atomic']))
+        upload_file.write('|-\n')
+
+        # fill up file with data
+        for package in packages:
+            dict_pack = eval(package)
+
+            upload_file.write('!style="text-align:left;" scope="row" | {}\n'.format(dict_pack['name']))
+
+            upload_file.write(format_color(dict_pack) + format_test_yml(dict_pack['dist-git']))  # dist-git tests.yml
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['classic']))  # dist-git classic
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['container']))  # dist-git container
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['test_tags']['atomic']))  # dist-git atomic
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['dist-git']['missing']))  # dist-git missing
+            upload_file.write(format_color(dict_pack) + format_pending(dict_pack['dist-git']['pending']))  # dist-git pending
+
+            upload_file.write(format_color(dict_pack) + format_test_yml(dict_pack['upstreamfirst']))  # upstreamfirst tests.yml
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['classic']))  # upstreamfirst classic
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['container']))  # upstreamfirst container
+            upload_file.write('{}| {}\n'.format(format_color(dict_pack), dict_pack['upstreamfirst']['test_tags']['atomic']))  # upstreamfirst atomic
+
+            upload_file.write('|-\n')
+
+        upload_file.write('|} \n')
+    upload_file.close()
+
+
+def format_pending(pending):
+    """Updates pending cell if package is on pending
+
+    Args:
+        pending (dict): pending is dict_pack['dist-git']['pending']
+
+    Returns:
+        empty cell or cell with info
+    """
+    if pending['status'] == '':
+        return '| \n'
+    else:
+
+        user_str = pending['user']['name'] + ' ' + pending['user']['fullname']
+        return '|[[{}|YES]] [{} _*] \n'.format(user_str, pending['url'])
+
+
+def format_test_yml(test_yml):
+    """
+    test_yml is dict_pack['upstreamfirst']
+    or
+    test_yml is dict_pack['dist-git']
+    """
+    if test_yml['package_url']:
+        return '| [{} {}]\n'.format(test_yml['package_url'], test_yml['test_yml'])
+    else:
+        return '| \n'
+
+
+def format_color(dict_pack):
+    """Function formats cell color to #7FFF00" 'Green' if package ready for porting"""
+
+    if dict_pack['dist-git']['pending']['status'] == '' \
+            and dict_pack['dist-git']['test_yml'] == '' \
+            and dict_pack['upstreamfirst']['test_yml'] != '':
+        return '! style="background-color:#7FFF00"'
+    return ''
+
+
+def get_packages_statistic(packages):
+    """Function gets packages statistic
+
+    Args:
+        packages (list): list of packages
+
+    Returns:
+        statistic_json (dict): packages statistic
+    """
+    statistic_json = {'total': '',
+                      'dist-git': {
+                          'test_yml': '',
+                          'missing': '',
+                          'pending': '',
+                          'test_tags': {'classic': '', 'container': '', 'atomic': ''}},
+                      'upstreamfirst': {'test_yml': '',
+                                        'test_tags': {'classic': '', 'container': '', 'atomic': ''}}}
+    total_packages = len(packages)
+    statistic_json['total'] = str(total_packages)
+    statistic_json['dist-git']['test_yml'] = _get_packages_number_in_category(total_packages, packages,
+                                                                              'dist-git', 'test_yml')
+    statistic_json['dist-git']['missing'] = _get_packages_number_in_category(total_packages, packages,
+                                                                             'dist-git', 'missing')
+    statistic_json['dist-git']['pending'] = _get_packages_number_in_category(total_packages, packages,
+                                                                             'dist-git', 'pending', 'status')
+    statistic_json['dist-git']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                          'dist-git',
+                                                                                          'test_tags', 'classic')
+    statistic_json['dist-git']['test_tags']['container'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                            'dist-git',
+                                                                                            'test_tags', 'container')
+    statistic_json['dist-git']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                         'dist-git',
+                                                                                         'test_tags', 'atomic')
+
+    statistic_json['upstreamfirst']['test_yml'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                   'upstreamfirst', 'test_yml')
+    statistic_json['upstreamfirst']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                               'upstreamfirst',
+                                                                                               'test_tags', 'classic')
+    statistic_json['upstreamfirst']['test_tags']['container'] = _get_packages_number_in_category(total_packages,
+                                                                                                 packages,
+                                                                                                 'upstreamfirst',
+                                                                                                 'test_tags',
+                                                                                                 'container')
+    statistic_json['upstreamfirst']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                              'upstreamfirst',
+                                                                                              'test_tags', 'atomic')
+    return statistic_json
+
+
+def _get_packages_number_in_category(total_packages, all_packages, *args):
+    """
+    Function returns updated statistic_json from the get_packages_statistic()
+    Args:
+        total_packages (int): total packages
+        all_packages (list of jsons): all packages
+
+    Returns:
+        formatted_string (all_packages):
+            for example: '48 (42%)'
+    """
+    number_in_category = 0
+    for item in all_packages:
+        dict_pack = eval(item)
+
+        if len(args) == 2:
+            if 'YES' in dict_pack[args[0]][args[1]]:
+                number_in_category += 1
+        else:
+            if 'YES' in dict_pack[args[0]][args[1]][args[2]]:
+                number_in_category += 1
+    formatted_string = "{} ({}%)".format(number_in_category, round((100*number_in_category)/total_packages))
+    return formatted_string
+
+
+if __name__ == '__main__':
+    exit(0)
diff --git a/statictic_page/new_page/requirements.txt b/statictic_page/new_page/requirements.txt
new file mode 100644
index 0000000..3866aaa
--- /dev/null
+++ b/statictic_page/new_page/requirements.txt
@@ -0,0 +1,2 @@
+requests
+mwclient
\ No newline at end of file
diff --git a/statictic_page/new_page/runner.py b/statictic_page/new_page/runner.py
new file mode 100644
index 0000000..4396671
--- /dev/null
+++ b/statictic_page/new_page/runner.py
@@ -0,0 +1,80 @@
+from tools import *
+
+from generate_file import generate_file_upload_on_wiki
+
+dist_git_api_key = config.dist_git_api_key
+
+UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL
+DIST_GIT_URL = config.DIST_GIT_URL
+
+package_json = {'name': '',
+                'dist-git': {
+                    'package_url': '',
+                    'test_yml': '',
+                    'missing': '',
+                    'pending': {'status': '', 'url': '', 'user': {}},
+                    'test_tags': {'classic': '', 'container': '', 'atomic': ''}},
+                'upstreamfirst': {'test_yml': '',
+                                  'package_url': '',
+                                  'test_tags': {'classic': '', 'container': '', 'atomic': ''}}}
+
+
+def wrapper_for_get_pull_requests(package):
+    """Function updates package_json with correct information"""
+
+    raw_text = get_pull_requests(DIST_GIT_URL, package, dist_git_api_key)
+    result = manage_pull_request(raw_text)
+    package_json['name'] = package
+
+    if result is not None:
+        package_json['dist-git']['pending']['url'] = result['url']
+        package_json['dist-git']['pending']['user'] = result['user']
+        package_json['dist-git']['pending']['status'] = 'YES'
+    else:
+        package_json['dist-git']['pending']['url'] = ''
+        package_json['dist-git']['pending']['user'] = ''
+        package_json['dist-git']['pending']['status'] = ''
+
+    # Get upstreamfirst test-tags
+    upstream_test_tags = handle_test_tags(UPSTREAMFIRST_URL, package)
+    upstream_test_tags = test_tags_to_dict(upstream_test_tags)
+    package_json['upstreamfirst']['test_tags'] = upstream_test_tags
+
+    upstream_url_to_test_yml = get_url_to_test_yml(UPSTREAMFIRST_URL, package)
+    if check_if_test_yml_exists(upstream_url_to_test_yml):
+        package_json['upstreamfirst']['test_yml'] = 'YES'
+        package_json['upstreamfirst']['package_url'] = upstream_url_to_test_yml
+    else:
+        package_json['upstreamfirst']['test_yml'] = ''
+        package_json['upstreamfirst']['package_url'] = ''
+
+    # Get dist-git test-tags
+    dist_git_test_tags = handle_test_tags(DIST_GIT_URL, package)
+    dist_git_test_tags = test_tags_to_dict(dist_git_test_tags)
+    package_json['dist-git']['test_tags'] = dist_git_test_tags
+
+    dist_git_url_to_test_yml = get_url_to_test_yml(DIST_GIT_URL, package)
+    if check_if_test_yml_exists(dist_git_url_to_test_yml):
+        package_json['dist-git']['test_yml'] = 'YES'
+        package_json['dist-git']['package_url'] = dist_git_url_to_test_yml
+    else:
+        package_json['dist-git']['test_yml'] = ''
+        package_json['dist-git']['package_url'] = ''
+
+    # Write results to the file
+    write_results_to_the_file(package_json, config.FILE_PACKAGES)
+
+
+if __name__ == '__main__':
+    # Prepare environment
+    delete_files_before_running([config.FILE_PACKAGES, config.FILE_UPLOAD_2_WIKI])
+
+    # Get all open pull requests
+    projects = get_all_projects(UPSTREAMFIRST_URL, upstreamfirst_api_key)
+    all_packages = get_projects_names(projects)
+
+    for package in all_packages:
+        wrapper_for_get_pull_requests(package)
+
+    generate_file_upload_on_wiki(config.FILE_PACKAGES)
+    exit(0)
diff --git a/statictic_page/new_page/tools.py b/statictic_page/new_page/tools.py
new file mode 100644
index 0000000..379fdd1
--- /dev/null
+++ b/statictic_page/new_page/tools.py
@@ -0,0 +1,272 @@
+import os
+import re
+import requests
+
+import config
+
+
+upstreamfirst_api_key = config.upstreamfirst_api_key
+dist_git_api_key = config.dist_git_api_key
+
+UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL
+DIST_GIT_URL = config.DIST_GIT_URL
+
+
+def get_pull_requests(site_url, package, api_key):
+    """Get pull requests from site using API
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+        api_key (string): secret API key
+
+    Returns:
+        response (json): info about PR
+    """
+    site = site_url + 'api/0/rpms/' + package + '/pull-requests'
+
+    response = requests.get(site, headers={'Authorization': 'access_token {}'.format(api_key)})
+    return response.json()
+
+
+def manage_pull_request(response_json):
+    """Checks if PR corresponds to the test PR and open.
+
+    Args:
+        response_json (json): info about PR
+
+    Returns:
+         (json): {'user': <username>, 'url': <pull_req_url>}
+    """
+    try:
+        if response_json['total_requests'] > 0:
+            for request in response_json['requests']:
+                if ('Add CI tests' or 'test' in request['title']) and (request['status'] == 'Open'):
+                    pull_req_url = DIST_GIT_URL + request['project']['url_path'] + '/pull-requests'
+                    return {'user': request['user'], 'url': pull_req_url}
+
+    except Exception:
+        # {'error': 'Project not found', 'error_code': 'ENOPROJECT'}
+        return
+
+
+def get_all_projects(site_url, api_key):
+    """Get all projects from site using API
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        api_key (string): secret API key
+
+    Returns:
+        response (json): all projects
+    """
+    site = site_url + 'api/0/projects'
+
+    response = requests.get(site, headers={'Authorization': 'access_token {}'.format(api_key)})
+    return response.json()
+
+
+def get_projects_names(json_response):
+    """Get projects names from the all projects
+
+    Args:
+        json_response (json): all projects
+
+    Returns:
+        projects_names (list of strings) names of projects
+    """
+
+    projects_names = []
+
+    for project in json_response['projects']:
+        project_name = project['fullname']
+        if 'forks/' not in project_name:
+            projects_names.append(project_name)
+
+    return projects_names
+
+
+def get_projects_url_patches(json_response):
+    """Get projects url patches"""
+
+    projects_url_patches = []
+
+    for project in json_response['projects']:
+        project_name = project['url_path']
+
+        projects_url_patches.append(project_name)
+
+    return projects_url_patches
+
+
+def get_test_yaml_from_site(site_url, package, test_file='tests.yml'):
+    """Get test.yaml from the site
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+        test_file (string): file to get from site.
+
+    Returns:
+        test.yaml (raw string)
+    """
+
+    if 'upstreamfirst' in site_url:
+        test_file_url = site_url + package + '/raw/master/f/' + test_file
+    elif 'fedoraproject' in site_url:
+        test_file_url = site_url + '/rpms/' + package + '/raw/master/f/tests/' + test_file
+    else:
+        return
+
+    response = requests.get(test_file_url)
+    return response.text
+
+
+def get_url_to_test_yml(site_url, package):
+    """Get url to the test.yml file
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+
+    Returns:
+        url (string)
+    """
+    if 'upstreamfirst' in site_url:
+        test_file_url = site_url + package + '/blob/master/f/tests.yml'
+    elif 'fedoraproject' in site_url:
+        test_file_url = site_url + 'rpms/' + package + '/blob/master/f/tests/tests.yml'
+    else:
+        return
+
+    return test_file_url
+
+
+def check_if_test_yml_exists(test_file_url):
+    """Checks if tesy.yml exists
+
+    Args:
+        test_file_url (string): url of the site
+
+    Returns:
+        True/False (bool) if response.status_code
+    """
+    response = requests.get(test_file_url)
+    if response.status_code == 200:
+        return True
+    else:
+        return False
+
+
+def get_test_tags(raw_text):
+    """Just returns existed test-tags
+
+    Args:
+        raw_text (string): raw text output from the requests
+
+    Returns:
+        tags (list if strings)
+    """
+    test_tags = []
+
+    for tag in ['classic', 'container', 'atomic']:
+        if tag in raw_text:
+            test_tags.append(tag)
+
+    return test_tags
+
+
+def handle_test_tags(site_url, package):
+    """Function grts new patch to the test.yaml if existing test.yaml includes test file
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+
+    Returns:
+        tags (list of strings)
+    """
+
+    raw_text = get_test_yaml_from_site(site_url, package)
+
+    if 'Page not found' in raw_text:
+        return []
+
+    test_tags = get_test_tags(raw_text)
+
+    if not test_tags:
+        new_test_file = re.findall(r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_text)
+
+        if new_test_file:
+            raw_text = get_test_yaml_from_site(site_url, package, new_test_file[-1])
+            test_tags = get_test_tags(raw_text)
+
+    return test_tags
+
+
+def test_tags_to_dict(test_tags):
+    """Convert test-tags list to the dictionary"""
+
+    tags_dict = {'classic': '', 'container': '', 'atomic': ''}
+    try:
+        for tag in test_tags:
+            tags_dict[tag] = 'YES'
+
+    except Exception.KeyError:
+        pass
+    return tags_dict
+
+
+def get_list_of_packages_from_the_file(file):
+    """Reads list of packages from a file
+
+    Args:
+        file (file): file to get packages from
+
+    Returns:
+        packages (list of packages)
+    """
+    with open(file) as packages_file:
+        return packages_file.read().splitlines()
+
+
+def write_results_to_the_file(result, filename='results.txt'):
+    """Write results to the file
+
+    Args:
+        result (json): package's info
+        filename (string): name of the file to write packages
+
+    Returns:
+        packages (list of packages)
+    """
+    with open(filename, 'a') as my_file:
+        my_file.write('{}\n'.format(result))
+
+    my_file.close()
+    return filename
+
+
+def delete_files_before_running(files_to_delete):
+    """Function deletes (cleanup) files with data
+
+    Args:
+        files_to_delete (list): list of files to delete
+
+    Returns:
+        (None) if everything is alright
+    """
+    for file_name in files_to_delete:
+        try:
+            os.remove(file_name)
+        except OSError:
+            pass
+
+
+if __name__ == '__main__':
+    exit(0)
diff --git a/statictic_page/new_page/upload_to_wiki.py b/statictic_page/new_page/upload_to_wiki.py
new file mode 100644
index 0000000..5c1d6e2
--- /dev/null
+++ b/statictic_page/new_page/upload_to_wiki.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import mwclient
+import os
+
+ua = 'MyWikiTool/0.2 run by User:FedoraUser'
+site = mwclient.Site('fedoraproject.org', clients_useragent=ua)
+# site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS'))
+site.login('sturivny', 'VgaZsxPWrMnCzTWm')
+page = site.pages['CI/Tests/new-stat']
+
+page.exists
+
+with open('upload_file.txt', 'r') as myfile:
+    text = myfile.read()
+page.save(text, 'Current status.')
diff --git a/statictic_page/stable_page/koji-stats b/statictic_page/stable_page/koji-stats
new file mode 100755
index 0000000..dd51f99
--- /dev/null
+++ b/statictic_page/stable_page/koji-stats
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+#
+# Copyright Red Hat Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# This file is part of Cockpit.
+#
+
+UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/"
+UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/"
+
+DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/"
+
+import argparse
+import json
+import os
+import random
+import sys
+import tempfile
+import time
+import shutil
+import subprocess
+
+import urllib.request
+import urllib.parse
+
+DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/"
+
+def main():
+    parser = argparse.ArgumentParser(description="Gather koji stats")
+    parser.add_argument("--scratch", action="store", default="/tmp", help="Scratch directory")
+    opts = parser.parse_args()
+
+    stats = { }
+    directory = tempfile.mkdtemp(prefix="koji-stats", dir=opts.scratch)
+
+    # Dist git
+    repos = repositories()
+
+    offset = random.randint(0, len(repos))
+    while True:
+        offset = (offset + 1) % len(repos)
+        project = repos[offset]
+
+        url = urllib.parse.urljoin(DISTGIT_BASE, project)
+        repo = clone(url, directory)
+        if not repo:
+            continue
+
+        start = time.time()
+        local(repo)
+        duration = int(time.time() - start)
+        log("mock", project, str(duration))
+
+        start = time.time()
+        koji(repo)
+        duration = int(time.time() - start)
+        log("koji", project, str(duration))
+
+        start = time.time()
+        koji(repo, "x86_64")
+        duration = int(time.time() - start)
+        log("koji-x86_64", project, str(duration))
+
+        shutil.rmtree(repo)
+
+    return 0
+
+def repositories():
+    results = [ ]
+    base = os.path.dirname(__file__)
+    with open(os.path.join(base, "repos"), "rb") as f:
+        while True:
+            line = f.readline()
+            if not line:
+                break
+            line = line.strip()
+            if line:
+                results.append(line.decode("utf-8"))
+    return results
+
+def log(self, *args):
+    line = "\t".join(args) + "\n"
+    sys.stdout.write(line)
+    with open(os.path.join(os.path.dirname(__file__), "koji-stats.log"), 'a') as f:
+        f.write(line)
+
+def clone(url, directory):
+    target = os.path.join(directory, os.path.basename(url))
+    while os.path.exists(target):
+        target += ".x"
+    try:
+        subprocess.check_call(["git", "clone", url, target], cwd=directory, stdout=sys.stderr.fileno())
+    except subprocess.CalledProcessError:
+        return None
+    subprocess.check_call(["git", "checkout", "f26"], cwd=target, stdout=sys.stderr.fileno())
+    return target
+
+def koji(directory, arch=None):
+    if arch:
+        cmd = ["fedpkg", "--scratch", "--arches", arch, "build"]
+    else:
+        cmd = ["fedpkg", "--scratch", "build"]
+    subprocess.check_call(cmd, cwd=directory, stdout=sys.stderr.fileno())
+
+def local(directory):
+    subprocess.check_call(["fedpkg", "mockbuild"], cwd=directory, stdout=sys.stderr.fileno())
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/statictic_page/stable_page/publish2wiki.py b/statictic_page/stable_page/publish2wiki.py
new file mode 100644
index 0000000..86ac933
--- /dev/null
+++ b/statictic_page/stable_page/publish2wiki.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+# Purpose: publish output of ./test-stats to Fedora Wiki. This script is
+# scheduled on Centos OpenShift: apps.ci.centos.org. Dependency are installed
+# as a part of Job definition. data.txt comes from ./test-stats output.
+
+import mwclient
+import os
+
+ua = 'MyWikiTool/0.2 run by User:FedoraUser'
+site = mwclient.Site('fedoraproject.org', clients_useragent=ua)
+site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS'))
+page = site.pages['CI/Tests/stat']
+page.exists
+with open('data.txt', 'r') as myfile:
+    text = myfile.read()
+page.save(text, 'Current status.')
diff --git a/statictic_page/stable_page/repos b/statictic_page/stable_page/repos
new file mode 100644
index 0000000..b21aa98
--- /dev/null
+++ b/statictic_page/stable_page/repos
@@ -0,0 +1,288 @@
+acl
+atomic
+atomic-devmode
+attr
+audit
+authconfig
+avahi
+basesystem
+bash
+bash-completion
+bind
+binutils
+biosdevname
+boost
+bridge-utils
+bzip2
+ca-certificates
+c-ares
+ceph
+checkpolicy
+chkconfig
+chrony
+cloud-init
+cloud-utils
+cockpit
+conntrack-tools
+container-selinux
+coreutils
+cpio
+cracklib
+criu
+cronie
+crontabs
+cryptsetup
+cups
+curl
+cyrus-sasl
+dbus
+dbus-glib
+dbus-python
+device-mapper
+device-mapper-multipath
+device-mapper-persistent-data
+dhcp
+diffutils
+dmidecode
+dnf
+dnsmasq
+docker
+docker-client
+docker-latest
+dracut
+e2fsprogs
+efibootmgr
+efivar
+etcd
+ethtool
+expat
+fcgi
+file
+filesystem
+findutils
+fipscheck
+flannel
+freetype
+fuse
+gawk
+gdbm
+GeoIP
+gettext
+glib2
+glibc
+glib-networking
+glusterfs
+gmp
+gnupg2
+gnutls
+gobject-introspection
+gomtree
+gperftools
+gpgme
+grep
+groff
+grub2
+grubby
+gsettings-desktop-schemas
+gssproxy
+gzip
+hardlink
+hostname
+initscripts
+iproute
+iptables
+iputils
+irqbalance
+iscsi-initiator-utils
+jansson
+json-glib
+kernel
+kexec-tools
+keyutils
+kmod
+krb5
+kubernetes
+less
+libaio
+libarchive
+libassuan
+libcap
+libcap-ng
+libcgroup
+libcollection
+libcroco
+libdb
+libdhash
+libedit
+libevent
+libffi
+gcc
+libgcrypt
+libgpg-error
+icu
+libidn
+libini_config
+ipa
+libldb
+libmnl
+libmodman
+libndp
+libnetfilter_conntrack
+libnetfilter_cthelper
+libnetfilter_cttimeout
+libnetfilter_queue
+libnfnetlink
+libnfsidmap
+libnl
+libnl3
+libpath_utils
+libpcap
+libproxy
+libpwquality
+libref_array
+librepo
+libreport
+libseccomp
+libselinux
+libsemanage
+libsepol
+libsolv
+libsoup
+libssh2
+sssd
+libtalloc
+libtasn1
+libtdb
+libtevent
+libtirpc
+libunistring
+libunwind
+libuser
+libutempter
+uuid
+libverto
+libxml2
+libyaml
+linux-firmware
+logrotate
+lttng-ust
+lua
+lvm2
+lzo
+m2crypto
+make
+mdadm
+mokutil
+mozjs17
+nano
+ncurses
+nettle
+net-tools
+NetworkManager
+newt
+nfs-utils
+nmap
+nspr
+nss
+nss-altfiles
+nss-softokn
+nss-util
+oci-register-machine
+oci-systemd-hook
+openldap
+openssh
+openssl
+os-prober
+ostree
+p11-kit
+pam
+passwd
+pciutils
+pcre
+pinentry
+pkgconfig
+policycoreutils
+polkit
+polkit-pkla-compat
+popt
+ppp
+procps-ng
+protobuf-c
+pth
+pygpgme
+pyliblzma
+python
+python-backports
+python-backports-ssl_match_hostname
+python-chardet
+python-configobj
+python-dateutil
+python-decorator
+python-dmidecode
+python-docker-py
+python-docker-pycreds
+python-ethtool
+python-iniparse
+python-ipaddress
+python-IPy
+python-jsonpatch
+python-jsonpointer
+python-prettytable
+python-pycurl
+python-pyudev
+python-requests
+python-rhsm
+python-setuptools
+python-six
+python-slip
+python-urlgrabber
+python-urllib3
+python-websocket-client
+pyxattr
+PyYAML
+quota
+readline
+rpcbind
+rpm
+rpm-ostree
+rsync
+runc
+samba
+sed
+selinux-policy
+setools
+setup
+sg3_utils
+shadow-utils
+shared-mime-info
+shim
+skopeo
+slang
+snappy
+socat
+sqlite
+sssd
+subscription-manager
+sudo
+systemd
+sysvinit
+tar
+tcp_wrappers
+tmux
+trousers
+tuned
+tzdata
+usermode
+userspace-rcu
+ustr
+util-linux
+vim
+virt-what
+which
+wpa_supplicant
+xfsprogs
+xz
+yajl
+yum
+yum-metadata-parser
+zlib
diff --git a/statictic_page/stable_page/requirements.txt b/statictic_page/stable_page/requirements.txt
new file mode 100644
index 0000000..df27baa
--- /dev/null
+++ b/statictic_page/stable_page/requirements.txt
@@ -0,0 +1 @@
+mwclient>=0.8.6
\ No newline at end of file
diff --git a/statictic_page/stable_page/run-tests b/statictic_page/stable_page/run-tests
new file mode 100755
index 0000000..dc21486
--- /dev/null
+++ b/statictic_page/stable_page/run-tests
@@ -0,0 +1,20 @@
+#!/bin/sh -euf
+
+# Prepare atomic host to test, report and artifacts dirs
+export TEST_SUBJECTS="$PWD/atomic.qcow2"
+test -f $TEST_SUBJECTS || curl -o $TEST_SUBJECTS http://artifacts.ci.centos.org/fedora-atomic/f26/images/latest-atomic.qcow2
+rm -rf report.txt artifacts/ clone/
+
+# Go through each project in upstreamfirst, clone it and put pass/fail in report.txt
+curl https://upstreamfirst.fedorainfracloud.org/api/0/projects?short=true | grep '"fullname"' | grep -v '/' | cut -d '"' -f 4 | while read repo; do
+	git clone https://upstreamfirst.fedorainfracloud.org/$repo clone/$repo
+	export ANSIBLE_INVENTORY=$(test -e $repo/inventory && echo clone/$repo/inventory || echo /usr/share/ansible/inventory)
+	export TEST_ARTIFACTS="$PWD/artifacts/$repo"
+	if ! ansible-playbook -i localhost, --list-tags clone/$repo/tests.yml | grep atomic; then
+		printf "$repo skip\n" >> report.txt
+	elif ansible-playbook -t atomic -e "subjects=$TEST_SUBJECTS" -e "artifacts=$TEST_ARTIFACTS" clone/$repo/tests.yml; then
+		printf "$repo pass\n" >> report.txt
+	else
+		printf "$repo fail\n" >> report.txt
+	fi
+done
diff --git a/statictic_page/stable_page/run-tests-parallel b/statictic_page/stable_page/run-tests-parallel
new file mode 100755
index 0000000..ccb7de5
--- /dev/null
+++ b/statictic_page/stable_page/run-tests-parallel
@@ -0,0 +1,27 @@
+#!/bin/sh -euf
+
+# Prepare atomic host to test, report and artifacts dirs
+export TEST_SUBJECTS="$PWD/atomic.qcow2"
+test -f $TEST_SUBJECTS || curl -o $TEST_SUBJECTS http://artifacts.ci.centos.org/fedora-atomic/f26/images/latest-atomic.qcow2
+rm -rf repo/ report/
+mkdir report/ repo/
+
+run() {
+	git clone https://upstreamfirst.fedorainfracloud.org/$1 repo/$1
+	inventory=$(test -e $1/inventory && echo repo/$1/inventory || echo /usr/share/ansible/inventory)
+	export TEST_ARTIFACTS="$PWD/repo/artifacts/$1"
+	if ! ansible-playbook --list-tags repo/$1/tests.yml | grep atomic; then
+		printf "$1 skip\n" >> report/$1.txt
+	elif ansible-playbook -i $inventory -t atomic -e "subjects=$TEST_SUBJECTS" -e "artifacts=$TEST_ARTIFACTS" repo/$1/tests.yml; then
+		printf "$1 pass\n" >> report/$1.txt
+	else
+		printf "$1 fail\n" >> report/$1.txt
+	fi
+}
+
+# Go through each project in upstreamfirst, clone it and put pass/fail in report.txt
+curl https://upstreamfirst.fedorainfracloud.org/api/0/projects?short=true | grep '"fullname"' | grep -v '/' | cut -d '"' -f 4 | while read repo; do
+	sleep 3
+	if test $(jobs | wc -l) -gt 16; then wait; fi
+	run $repo &
+done
diff --git a/statictic_page/stable_page/test-stats b/statictic_page/stable_page/test-stats
new file mode 100755
index 0000000..8e70d1e
--- /dev/null
+++ b/statictic_page/stable_page/test-stats
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright Red Hat Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# This file is part of Cockpit.
+#
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+import shutil
+import subprocess
+
+import urllib.request
+import urllib.parse
+
+
+UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/"
+UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/"
+
+DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/"
+
+
+def retrieve(url):
+    return urllib.request.urlopen(url).read()
+
+
+def projects():
+    url = urllib.parse.urljoin(UPSTREAM_PAGURE, "projects")
+    for project in json.loads(retrieve(url))["projects"]:
+        if not project.get("parent"):
+            yield project["name"]
+
+
+def repos():
+    base = os.path.dirname(__file__)
+    with open(os.path.join(base, "repos"), "rb") as f:
+        while True:
+            line = f.readline()
+            if not line:
+                break
+            line = line.strip()
+            if line:
+                yield line.decode("utf-8")
+
+
+def clone(url, directory):
+    target = os.path.join(directory, os.path.basename(url))
+    while os.path.exists(target):
+        target += ".x"
+    try:
+        subprocess.check_call(["git", "clone", url, target], cwd=directory, stdout=sys.stderr.fileno())
+    except subprocess.CalledProcessError:
+        return None
+    return target
+
+
+def count(stats, field, project):
+    if field not in stats:
+        stats[field] = [ ]
+    stats[field].append(project)
+
+
+def tags(repo, standard):
+    try:
+        playbook = standard and "tests.yml" or "test_local.yml"
+        environ = os.environ.copy()
+        environ["LC_ALL"] = "C"
+        output = subprocess.check_output([ "ansible-playbook", "--list-tags", playbook ], env=environ, cwd=repo)
+    except subprocess.CalledProcessError:
+        output = b""
+
+    for line in output.decode("utf-8").split("\n"):
+        if "TASK TAGS" in line:
+            name, unused, tags = line.partition(":")
+            for tag in tags.strip(" []").split(","):
+                yield tag.strip()
+
+
+def gather(repo, project, stats):
+
+    # Standard entry point
+    path = os.path.join(repo, "tests.yml")
+    standard = os.path.exists(path)
+    if standard:
+        count(stats, "tests.yml", project)
+
+    # Check if we have an old style test_cloud.yml file
+    path = os.path.join(repo, "test_cloud.yml")
+    if os.path.exists(path):
+        count(stats, "atomic", project)
+    elif "atomic" in tags(repo, standard):
+        count(stats, "atomic", project)
+
+    if os.path.exists(os.path.join(repo, "test_cloud.yml")) or \
+       os.path.exists(os.path.join(repo, "test_local.yml")) or \
+       os.path.exists(os.path.join(repo, "test_docker.yml")):
+        count(stats, "legacy", project)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Gather stats about tests in dist-git')
+    opts = parser.parse_args()
+
+    stats = {}
+    directory = tempfile.mkdtemp(prefix="test-stats")
+
+    # Dist git
+    for project in repos():
+        url = urllib.parse.urljoin(DISTGIT_BASE, project)
+        repo = clone(url, directory)
+        if repo:
+            sub = os.path.join(repo, "tests")
+            count(stats, "distgit", project)
+            if os.path.exists(sub):
+                count(stats, "distgit-tests", project)
+                gather(sub, project, stats)
+        else:
+            count(stats, "missing", project)
+
+    # Upstream first projects
+    for project in projects():
+        url = urllib.parse.urljoin(UPSTREAM_BASE, project)
+        repo = clone(url, directory)
+        if repo:
+            count(stats, "upstream", project)
+            gather(repo, project, stats)
+
+    shutil.rmtree(directory)
+
+    for name in list(stats.keys()):
+        stats["{0}-length".format(name)] = len(stats[name])
+    json.dump(stats, sys.stdout, indent=4)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

From 9d2f476c30e86435f21b4100a5912887074a316e Mon Sep 17 00:00:00 2001
From: Sergey Turivniy <turivniy@gmail.com>
Date: Jan 10 2018 17:56:59 +0000
Subject: [PATCH 2/2] Add functionality to get packages from URL


---

diff --git a/statictic_page/stable_page/.gitignore b/statictic_page/stable_page/.gitignore
new file mode 100644
index 0000000..bcf1aa4
--- /dev/null
+++ b/statictic_page/stable_page/.gitignore
@@ -0,0 +1,2 @@
+*.txt
+.idea*
diff --git a/statictic_page/stable_page/bash-job.yml b/statictic_page/stable_page/bash-job.yml
new file mode 100644
index 0000000..7575532
--- /dev/null
+++ b/statictic_page/stable_page/bash-job.yml
@@ -0,0 +1,41 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: tests2wiki
+spec:
+  parallelism: 1
+  completions: 1
+  template:
+    metadata:
+      name: pi
+    spec:
+      containers:
+      - name: pi
+        image: python
+        command:
+            - bash
+            - -c
+            - |
+              cd /tmp
+              HOME=/tmp pip install --user virtualenv
+              PYTHONPATH="/tmp/.local/lib/python3.6/site-packages:$PYTHONPATH" /tmp/.local/bin/virtualenv  /tmp/v
+              source /tmp/v/bin/activate
+              pip install -r /tmp/fedora-ci-stats/requirements.txt
+              export GIT_COMMITTER_NAME=git
+              export GIT_COMMITTER_EMAIL=git@git.git
+              git clone --branch wikireport https://github.com/Andrei-Stepanov/fedora-ci-stats
+              git clone https://pagure.io/standard-test-roles.git
+              echo -e '[defaults]\nroles_path=/tmp/standard-test-roles/roles' > /tmp/ansible.cfg
+              export ANSIBLE_CONFIG=/tmp/ansible.cfg
+              pushd fedora-ci-stats
+              python test-stats.py --wikiout wdata.mw --reposlist repos
+              python test-stats.py --wikiout wdata_hp.mw --reposlist repos_hp
+              cat wdata.mw
+              cat wdata_hp.mw
+              #export WIKI_USER=<YOUR FEDORA FAS LOGIN>
+              #export WIKI_PASS=<YOUR FEDORA FASS PASS>
+              ./publish2wiki.py --filedoc wdata.mw --pagepath CI/Tests/stat
+              ./publish2wiki.py --filedoc wdata_hp.mw --pagepath CI/Tests/stat_hp
+      restartPolicy: OnFailure
+
+# vim: et ts=2 sw=2 ai
diff --git a/statictic_page/stable_page/config.py b/statictic_page/stable_page/config.py
new file mode 100644
index 0000000..bbc43c1
--- /dev/null
+++ b/statictic_page/stable_page/config.py
@@ -0,0 +1,12 @@
+UPSTREAMFIRST_URL = 'https://upstreamfirst.fedorainfracloud.org/'
+DIST_GIT_URL = 'https://src.fedoraproject.org/'
+
+FILE_UPLOAD_2_WIKI = 'upl2wiki.txt'
+FILE_PACKAGES = 'upstreamfirst_packages.txt'
+
+DEFAULT_URL='fedoraproject.org'
+UPSTREAM_PAGURE = "https://upstreamfirst.fedorainfracloud.org/api/0/"
+UPSTREAM_BASE = "https://upstreamfirst.fedorainfracloud.org/"
+DISTGIT_BASE = "https://src.fedoraproject.org/git/rpms/"
+
+IN_J2_WIKI_TEMPLATE = 'stat.j2'
\ No newline at end of file
diff --git a/statictic_page/stable_page/dump2wiki-cronjob.yml b/statictic_page/stable_page/dump2wiki-cronjob.yml
new file mode 100644
index 0000000..4e8ca51
--- /dev/null
+++ b/statictic_page/stable_page/dump2wiki-cronjob.yml
@@ -0,0 +1,45 @@
+apiVersion: batch/v2alpha1
+kind: CronJob
+metadata:
+  name: tests2wiki
+spec:
+  schedule: "11 9,18 * * *"
+  ConcurrencyPolicy: Replace
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            parent: "cronjobpi"
+        spec:
+          containers:
+          - name: wiki
+            image: python
+            command:
+                - bash
+                - -c
+                - |
+                  cd /tmp
+                  HOME=/tmp pip install --user virtualenv
+                  PYTHONPATH="/tmp/.local/lib/python3.6/site-packages:$PYTHONPATH" /tmp/.local/bin/virtualenv  /tmp/v
+                  source /tmp/v/bin/activate
+                  pip install -r /tmp/fedora-ci-stats/requirements.txt
+                  export GIT_COMMITTER_NAME=git
+                  export GIT_COMMITTER_EMAIL=git@git.git
+                  git clone --branch wikireport https://github.com/Andrei-Stepanov/fedora-ci-stats
+                  git clone https://pagure.io/standard-test-roles.git
+                  echo -e '[defaults]\nroles_path=/tmp/standard-test-roles/roles' > /tmp/ansible.cfg
+                  export ANSIBLE_CONFIG=/tmp/ansible.cfg
+                  pushd fedora-ci-stats
+                  python test-stats.py --wikiout wdata.mw --reposlist repos
+                  python test-stats.py --wikiout wdata_hp.mw --reposlist repos_hp
+                  cat wdata.mw
+                  cat wdata_hp.mw
+                  #export WIKI_USER=<YOUR FEDORA FAS LOGIN>
+                  #export WIKI_PASS=<YOUR FEDORA FASS PASS>
+                  ./publish2wiki.py --filedoc wdata.mw --pagepath CI/Tests/stat
+                  ./publish2wiki.py --filedoc wdata_hp.mw --pagepath CI/Tests/stat_hp
+          restartPolicy: Never
+
+
+# vim: et ts=2 sw=2 ai
diff --git a/statictic_page/stable_page/generate_file_upload_wiki.py b/statictic_page/stable_page/generate_file_upload_wiki.py
new file mode 100644
index 0000000..cfdc687
--- /dev/null
+++ b/statictic_page/stable_page/generate_file_upload_wiki.py
@@ -0,0 +1,124 @@
+import argparse
+import datetime
+import jinja2
+import os
+
+from tools import get_list_of_packages_from_the_file
+from config import IN_J2_WIKI_TEMPLATE
+
+
+def get_list_of_dicts(dict_of_strings):
+    result = []
+    for package in dict_of_strings:
+        result.append(eval(package))
+    return result
+
+
+def generate_file_upload_on_wiki(input_file, output_file):
+    """Generate file that will be uploaded on the wiki page
+
+    Args:
+        input_file (file): file with packages
+        output_file (file): file with packages in json format
+
+    Returns:
+        (file): file that will be uploaded on the wiki page
+    """
+    THIS_DIR = os.path.dirname(os.path.abspath(__file__))
+    packages = get_list_of_packages_from_the_file(input_file)
+    packages_statistic = get_packages_statistic(packages)
+    packages = get_list_of_dicts(packages)
+
+    j2_loader = jinja2.FileSystemLoader(THIS_DIR)
+    j2_env = jinja2.Environment(loader=j2_loader, trim_blocks=True)
+    template = j2_env.get_template(IN_J2_WIKI_TEMPLATE)
+
+    template_vars = {}
+    template_vars['updated'] = datetime.datetime.utcnow()
+    template_vars['total'] = packages_statistic
+    template_vars['pkgs'] = packages
+
+    with open(output_file, 'w') as mwfile:
+
+        for line in template.render(template_vars):
+            mwfile.write(line)
+
+
+def get_packages_statistic(packages):
+    """Function gets packages statistic
+
+    Args:
+        packages (list): list of packages
+
+    Returns:
+        statistic_json (dict): packages statistic
+    """
+    statistic_json = {'total': '',
+                      'distgit': {
+                          'test_yml': '',
+                          'missing': '',
+                          'pending': '',
+                          'test_tags': {'classic': '', 'container': '', 'atomic': ''}},
+                      'upstreamfirst': {'test_yml': '',
+                                        'test_tags': {'classic': '', 'container': '', 'atomic': ''}}}
+    total_packages = len(packages)
+    statistic_json['total'] = str(total_packages)
+    statistic_json['distgit']['test_yml'] = _get_packages_number_in_category(total_packages, packages,
+                                                                              'distgit', 'test_yml')
+    statistic_json['distgit']['missing'] = _get_packages_number_in_category(total_packages, packages,
+                                                                             'distgit', 'missing')
+    statistic_json['distgit']['pending'] = _get_packages_number_in_category(total_packages, packages,
+                                                                             'distgit', 'pending', 'status')
+    statistic_json['distgit']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                          'distgit',
+                                                                                          'test_tags', 'classic')
+    statistic_json['distgit']['test_tags']['container'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                            'distgit',
+                                                                                            'test_tags', 'container')
+    statistic_json['distgit']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                         'distgit',
+                                                                                         'test_tags', 'atomic')
+
+    statistic_json['upstreamfirst']['test_yml'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                   'upstreamfirst', 'test_yml')
+    statistic_json['upstreamfirst']['test_tags']['classic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                               'upstreamfirst',
+                                                                                               'test_tags', 'classic')
+    statistic_json['upstreamfirst']['test_tags']['container'] = _get_packages_number_in_category(total_packages,
+                                                                                                 packages,
+                                                                                                 'upstreamfirst',
+                                                                                                 'test_tags',
+                                                                                                 'container')
+    statistic_json['upstreamfirst']['test_tags']['atomic'] = _get_packages_number_in_category(total_packages, packages,
+                                                                                              'upstreamfirst',
+                                                                                              'test_tags', 'atomic')
+    return statistic_json
+
+
+def _get_packages_number_in_category(total_packages, all_packages, *args):
+    """
+    Function returns updated statistic_json from the get_packages_statistic()
+    Args:
+        total_packages (int): total packages
+        all_packages (list of jsons): all packages
+
+    Returns:
+        formatted_string (all_packages):
+            for example: '48 (42%)'
+    """
+    number_in_category = 0
+    for item in all_packages:
+        dict_pack = eval(item)
+
+        if len(args) == 2:
+            if dict_pack[args[0]][args[1]]:
+                number_in_category += 1
+        else:
+            if dict_pack[args[0]][args[1]][args[2]]:
+                number_in_category += 1
+    formatted_string = "{} ({}%)".format(number_in_category, round((100*number_in_category)/total_packages))
+    return formatted_string
+
+
+if __name__ == '__main__':
+    exit(0)
diff --git a/statictic_page/stable_page/publish2wiki.py b/statictic_page/stable_page/publish2wiki.py
old mode 100644
new mode 100755
index 86ac933..ba33973
--- a/statictic_page/stable_page/publish2wiki.py
+++ b/statictic_page/stable_page/publish2wiki.py
@@ -1,17 +1,78 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# Purpose: publish output of ./test-stats to Fedora Wiki. This script is
-# scheduled on Centos OpenShift: apps.ci.centos.org. Dependency are installed
-# as a part of Job definition. data.txt comes from ./test-stats output.
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2017
+# Author: Andrei Stepanov <astepano@redhat.com>
+#
+
+"""Purpose: Publish some document on MediaWiki.
+
+"""
 
-import mwclient
 import os
+import logging
+import mwclient
+import argparse
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+parser = argparse.ArgumentParser(
+    description='Publish a document on MediaWiki.')
+
+
+DEFAULT_URL='fedoraproject.org'
+
+parser.add_argument("--url", metavar='URL', help="Wiki URL. Default: %s"
+                    % DEFAULT_URL, default=DEFAULT_URL)
+
+parser.add_argument("--login", metavar='LOGIN',
+                    help=("Wiki user. "
+                          "If not set takes from Env variable: $WIKI_LOGIN"))
+
+parser.add_argument("--passw", metavar='PASSWORD',
+                    help=("Wiki user password. "
+                          "If not set takes from Env variable: $WIKI_PASS"))
+
+parser.add_argument("--filedoc", metavar='DOCUMENT',
+                    required=True, help="Path to a document file to publish.")
+
+parser.add_argument("--pagepath", metavar='PAGEPATH', required=True,
+                    help=("Path to Wiki page to be updated. "
+                          "Example: Section/Sub/Doc"))
+
+args = parser.parse_args()
+
+login = args.login
+if not login:
+    login = os.environ.get('WIKI_USER')
+
+passw = args.passw
+if not passw:
+    passw = os.environ.get('WIKI_PASS')
+
+logger.info("URL: %s", args.url)
+logger.info("LOGIN: %s", login)
+logger.info("DOCUMENT FILE: %s", args.filedoc)
+logger.info("WIKI PAGE: %s", args.pagepath)
 
 ua = 'MyWikiTool/0.2 run by User:FedoraUser'
-site = mwclient.Site('fedoraproject.org', clients_useragent=ua)
-site.login(os.environ.get('WIKI_USER'), os.environ.get('WIKI_PASS'))
-page = site.pages['CI/Tests/stat']
-page.exists
-with open('data.txt', 'r') as myfile:
-    text = myfile.read()
-page.save(text, 'Current status.')
+site = mwclient.Site(args.url, clients_useragent=ua)
+site.login(login, passw)
+page = site.pages[args.pagepath]
+if not page.exists:
+    logger.info("Page %s doesn't exist. Create a new one.", args.pagepath)
+with open(args.filedoc, 'r') as doc:
+    text = doc.read()
+page.save(text, 'Auto updated.')
diff --git a/statictic_page/stable_page/repos b/statictic_page/stable_page/repos
index b21aa98..270acd3 100644
--- a/statictic_page/stable_page/repos
+++ b/statictic_page/stable_page/repos
@@ -286,3 +286,55 @@ yajl
 yum
 yum-metadata-parser
 zlib
+# H&P components. Contact: Rachel Sibley
+babeltrace
+byacc
+cflow
+crypto-policies
+ctags
+dwz
+elfutils
+emacs
+fedora-logos
+fedora-release
+fedora-repos
+fedpkg-minimal
+flex
+fpc-srpm-macros
+ghc-srpm-macros
+gnat-srpm-macros
+go-srpm-macros
+guile
+hmaccalc
+isl
+ksh
+libatomic_ops
+libipt
+libksba
+libmetalink
+libmpc
+libpipeline
+libpng
+libpsl
+libsigsegv
+libtool
+lz4
+m4
+man-db
+mpfr
+ncompress
+nghttp2
+npth
+nss-pem
+ocaml-srpm-macros
+perl-srpm-macros
+psmisc
+publicsuffix-list
+python3
+python-rpm-macros
+qrencode
+qt5
+redhat-rpm-config
+sharutils
+texinfo
+unzip
diff --git a/statictic_page/stable_page/repos_hp b/statictic_page/stable_page/repos_hp
new file mode 100644
index 0000000..a9b7fdc
--- /dev/null
+++ b/statictic_page/stable_page/repos_hp
@@ -0,0 +1,322 @@
+acl
+alsa-lib
+attr
+audit
+augeas
+b43-openfwwf
+babeltrace
+basesystem
+bash
+bc
+bind99
+binutils
+biosdevname
+bluez
+btrfs-progs
+byacc
+bzip2
+ca-certificates
+c-ares
+ceph
+checkpolicy
+checksec
+chkconfig
+chrpath
+cmake
+cockpit
+compat-openssl10
+coreutils
+cpio
+cracklib
+crypto-policies
+cryptsetup
+curl
+cyrus-sasl
+dbus
+dbus-glib
+dbxtool
+deltarpm
+desktop-file-utils
+device-mapper-multipath
+dhcp
+diffutils
+ding-libs
+dmraid
+dnf
+docbook-dtds
+docbook-style-xsl
+dosfstools
+dracut
+dtc
+dwz
+e2fsprogs
+ebtables
+efibootmgr
+efivar
+elfutils
+emacs
+ethtool
+expat
+fbset
+fcoe-utils
+fedora-logos
+fedora-modular-release
+fedora-modular-repos
+fedora-repos
+fedpkg-minimal
+file
+filesystem
+findutils
+fipscheck
+flex
+fpc-srpm-macros
+freetype
+fuse
+gawk
+gc
+gcc
+gdb
+gdbm
+gdisk
+gettext
+ghc-srpm-macros
+glib2
+glibc
+glibc32
+glib-networking
+gmp
+gnat-srpm-macros
+gnupg
+gnupg2
+gnutls
+gobject-introspection
+go-srpm-macros
+gpart
+gpgme
+grep
+groff
+grub2
+grubby
+gsettings-desktop-schemas
+gssproxy
+guile
+gzip
+hfsplus-tools
+hfsutils
+hmaccalc
+hostname
+hwdata
+icu
+ima-evm-utils
+initscripts
+ipcalc
+iproute
+ipset
+iptables
+iputils
+iscsi-initiator-utils
+isl
+isns-utils
+jansson
+jbigkit
+json-c
+json-glib
+kbd
+kernel
+kexec-tools
+keyutils
+kmod
+krb5
+less
+libaio
+libarchive
+libassuan
+libatomic_ops
+libblockdev
+libcap
+libcap-ng
+libcgroup
+libcomps
+libconfig
+libcroco
+libdaemon
+libdb
+libdnf
+libedit
+libev
+libevent
+libffi
+libgcrypt
+libgpg-error
+libidn
+libidn2
+libipt
+libjpeg-turbo
+libksba
+libmetalink
+libmnl
+libmodman
+libmpc
+libnetfilter_conntrack
+libnfnetlink
+libnfsidmap
+libnl3
+libpcap
+libpciaccess
+libpeas
+libpipeline
+libpng
+libproxy
+libpwquality
+librepo
+libreport
+librtas
+libseccomp
+libsecret
+libselinux
+libsemanage
+libsepol
+libservicelog
+libsigsegv
+libsolv
+libssh
+libssh2
+libtalloc
+libtasn1
+libtevent
+libtiff
+libtirpc
+libtool
+libunistring
+libunwind
+libusb
+libusbx
+libuser
+libutempter
+libverto
+libvpd
+libxml2
+libxslt
+libyaml
+linux-firmware
+lksctp-tools
+lldpad
+logrotate
+lsof
+lsscsi
+lsvpd
+lttng-ust
+lua
+lvm2
+lz4
+lzo
+m4
+mactel-boot
+make
+man-db
+mcstrans
+mdadm
+memtest86+
+microdnf
+modulemd
+mokutil
+mpfr
+mtools
+multilib-rpm-config
+nano
+ncurses
+nettle
+nfs-utils
+nghttp2
+npth
+nspr
+nss
+nss-pem
+nss-softokn
+nss-util
+ocaml-srpm-macros
+openblas-srpm-macros
+openjpeg2
+openldap
+openssh
+openssl
+os-prober
+p11-kit
+pam
+parted
+passwd
+patch
+pciutils
+pcre
+pcre2
+perl-srpm-macros
+pkgconf
+plymouth
+policycoreutils
+polkit
+popt
+powerpc-utils
+ppc64-diag
+ppc64-utils
+procps-ng
+psmisc
+publicsuffix-list
+python3
+python-dateutil
+python-iniparse
+python-IPy
+python-pip
+python-rpm-macros
+python-setuptools
+python-six
+python-smartcols
+PyYAML
+qrencode
+qt5
+quota
+readline
+redhat-rpm-config
+rootfiles
+rpcbind
+rpm
+rsync
+rust-srpm-macros
+s390utils
+sed
+selinux-policy
+servicelog
+setools
+setup
+sg3_utils
+sgml-common
+sgpio
+shadow-utils
+shared-mime-info
+shim-signed
+snappy
+sqlite
+sudo
+syslinux
+systemd
+tar
+tcl
+tcp_wrappers
+texinfo
+time
+tmux
+tzdata
+uboot-tools
+unzip
+usermode
+userspace-rcu
+util-linux
+vim
+volume_key
+which
+words
+xfsprogs
+xmlrpc-c
+xz
+zip
+zlib
+zstd
diff --git a/statictic_page/stable_page/requirements.txt b/statictic_page/stable_page/requirements.txt
index df27baa..1587356 100644
--- a/statictic_page/stable_page/requirements.txt
+++ b/statictic_page/stable_page/requirements.txt
@@ -1 +1,5 @@
-mwclient>=0.8.6
\ No newline at end of file
+requests
+mwclient
+jinja2>=2.10
+ansible
+argparse
\ No newline at end of file
diff --git a/statictic_page/stable_page/stat.j2 b/statictic_page/stable_page/stat.j2
new file mode 100644
index 0000000..0768d5c
--- /dev/null
+++ b/statictic_page/stable_page/stat.j2
@@ -0,0 +1,73 @@
+== This page is automatically updated. Do not edit. ==
+
+Page was updated on: {{updated}} UTC
+
+{% raw %}
+{| class="wikitable sortable"
+{% endraw %}
+! style="text-align:left;" scope="col" rowspan="2" | Package
+! scope="col" colspan="6"| dist-git
+! scope="col" colspan="4"| upstreamfirst
+|-
+! scope="col" | tests.yml
+! scope="col" | classic
+! scope="col" | container
+! scope="col" | atomic
+! scope="col" | missing
+! scope="col" | pending PR
+! scope="col" | tests.yml
+! scope="col" | classic
+! scope="col" | container
+! scope="col" | atomic
+|-
+! style="text-align:left; font-weight: lighter;" scope="row" | Total known packages
+| style="text-align:center;" scope="row" colspan="10" | {{total.total}}
+|-
+! style="text-align:left; font-weight: lighter;" scope="row" | Packages in category
+|{{total.distgit.test_yml}}
+|{{total.distgit.test_tags.classic}}
+|{{total.distgit.test_tags.atomic}}
+|{{total.distgit.test_tags.container}}
+|{{total.distgit.missing}}
+|{{total.distgit.pending}}
+|{{total.upstreamfirst.test_yml}}
+|{{total.upstreamfirst.test_tags.classic}}
+|{{total.upstreamfirst.test_tags.container}}
+|{{total.upstreamfirst.test_tags.atomic}}
+{% for package in pkgs %}
+|-
+! style="text-align:left;" scope="row" | {{package.name}}
+|{% if package.distgit.test_yml %} yes {% else %} - {% endif %} [https://src.fedoraproject.org/rpms/{{package.name}}/tree/master *]
+|{% if package.distgit.test_tags.classic %} yes {% else %} - {% endif %}
+
+|{% if package.distgit.test_tags.container %} yes {% else %} - {% endif %}
+
+|{% if package.distgit.test_tags.atomic %} yes {% else %} - {% endif %}
+
+|{% if package.distgit.missing %} yes {% else %} - {% endif %}
+
+|{% if package.distgit.pending.status %} yes [https://src.fedoraproject.org/rpms/{{package.name}}/pull-requests *] {% else %} - {%- endif %}
+
+|{% if package.upstreamfirst.test_yml %} yes [https://upstreamfirst.fedorainfracloud.org/{{package.name}}/tree/master *] {% else %} - {%- endif %}
+
+|{% if package.upstreamfirst.test_tags.classic %} yes {% else %} - {% endif %}
+
+|{% if package.upstreamfirst.test_tags.container %} yes {% else %} - {% endif %}
+
+|{% if package.upstreamfirst.test_tags.atomic %} yes {% else %} - {% endif %}
+
+{% endfor %}
+{% raw %}
+|}
+{% endraw %}
+
+<!--
+
+    https://fedoraproject.org/wiki/Help:Wiki_syntax_and_markup
+    https://www.mediawiki.org/wiki/Help:Tables
+
+    -->
+
+<!--
+     vim: ft=xml
+     -->
\ No newline at end of file
diff --git a/statictic_page/stable_page/test-stats.py b/statictic_page/stable_page/test-stats.py
new file mode 100755
index 0000000..018a4cb
--- /dev/null
+++ b/statictic_page/stable_page/test-stats.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3.6
+# -*- coding: utf-8 -*-
+
+# Copyright Red Hat Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# This file is part of Cockpit.
+
+import argparse
+from tools import *
+
+from generate_file_upload_wiki import generate_file_upload_on_wiki
+
+
+UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL
+DIST_GIT_URL = config.DIST_GIT_URL
+
+package_json = {'name': '',
+                'distgit': {
+                    'package_url': '',
+                    'test_yml': '',
+                    'missing': '',
+                    'pending': {'status': '', 'url': '', 'user': {}},
+                    'test_tags': {'classic': '', 'container': '', 'atomic': ''}},
+                'upstreamfirst': {'test_yml': '',
+                                  'package_url': '',
+                                  'test_tags': {'classic': '', 'container': '', 'atomic': ''}}}
+
+
+def wrapper_for_get_pull_requests(package):
+    """Function updates package_json with correct information"""
+
+    raw_text = get_pull_requests(DIST_GIT_URL, package)
+    result = manage_pull_request(raw_text)
+    package_json['name'] = package
+
+    if result is not None:
+        package_json['distgit']['pending']['url'] = result['url']
+        package_json['distgit']['pending']['user'] = result['user']
+        package_json['distgit']['pending']['status'] = True
+    else:
+        package_json['distgit']['pending']['url'] = ''
+        package_json['distgit']['pending']['user'] = ''
+        package_json['distgit']['pending']['status'] = False
+
+    # Get upstreamfirst test-tags
+    upstream_test_tags = handle_test_tags(UPSTREAMFIRST_URL, package)
+    upstream_test_tags = test_tags_to_dict(upstream_test_tags)
+    package_json['upstreamfirst']['test_tags'] = upstream_test_tags
+
+    upstream_url_to_test_yml = get_url_to_test_yml(UPSTREAMFIRST_URL, package)
+    if check_if_test_yml_exists(upstream_url_to_test_yml):
+        package_json['upstreamfirst']['test_yml'] = True
+        package_json['upstreamfirst']['package_url'] = upstream_url_to_test_yml
+    else:
+        package_json['upstreamfirst']['test_yml'] = False
+        package_json['upstreamfirst']['package_url'] = ''
+
+    # Get distgit test-tags
+    dist_git_test_tags = handle_test_tags(DIST_GIT_URL, package)
+    dist_git_test_tags = test_tags_to_dict(dist_git_test_tags)
+    package_json['distgit']['test_tags'] = dist_git_test_tags
+
+    dist_git_url_to_test_yml = get_url_to_test_yml(DIST_GIT_URL, package)
+    if check_if_test_yml_exists(dist_git_url_to_test_yml):
+        package_json['distgit']['test_yml'] = True
+        package_json['distgit']['package_url'] = dist_git_url_to_test_yml
+    else:
+        package_json['distgit']['test_yml'] = False
+        package_json['distgit']['package_url'] = ''
+
+    # Write results to the file
+    write_results_to_the_file(package_json, config.FILE_PACKAGES)
+
+
+def get_projects_list_from_url(site_url):
+    # Get all open pull requests
+    projects = get_all_projects(UPSTREAMFIRST_URL)
+    all_packages = get_projects_names(projects)
+    return all_packages
+
+
+def create_file_with_packages_data(input_data):
+    # Create file with packages_data input data is a file or an URL. Filename is config.FILE_PACKAGES
+    if 'https://' in input_data:
+        all_packages = get_projects_list_from_url(input_data)
+    else:
+        all_packages = get_list_of_packages_from_the_file(input_data)
+
+    for package in all_packages:
+        wrapper_for_get_pull_requests(package)
+    return
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Gather stats about tests in dist-git')
+    parser.add_argument("--wikiout", metavar='FILE', default=None, help="Dump output to FILE in MediaWiki format.")
+    parser.add_argument("--reposlist",
+                        metavar='REPOSFILE',
+                        default=None,
+                        help="File or URL with repos. Could be whether 'https://upstreamfirst.fedorainfracloud.org/' "
+                             "or 'https://src.fedoraproject.org/'. One repo per line.",
+                        required=True)
+    opts = parser.parse_args()
+
+    # Prepare environment
+    delete_files_before_running([config.FILE_PACKAGES, opts.wikiout])
+
+    if opts.wikiout:
+        create_file_with_packages_data(opts.reposlist)
+        generate_file_upload_on_wiki(input_file=config.FILE_PACKAGES, output_file=opts.wikiout)
+    else:
+        create_file_with_packages_data(opts.reposlist)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/statictic_page/stable_page/tools.py b/statictic_page/stable_page/tools.py
new file mode 100644
index 0000000..f8551b6
--- /dev/null
+++ b/statictic_page/stable_page/tools.py
@@ -0,0 +1,267 @@
+import os
+import re
+import requests
+
+import config
+
+
+UPSTREAMFIRST_URL = config.UPSTREAMFIRST_URL
+DIST_GIT_URL = config.DIST_GIT_URL
+
+
+def get_pull_requests(site_url, package):
+    """Get pull requests from site using API
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+
+    Returns:
+        response (json): info about PR
+    """
+    site = site_url + 'api/0/rpms/' + package + '/pull-requests'
+
+    response = requests.get(site)
+    return response.json()
+
+
+def manage_pull_request(response_json):
+    """Checks if PR corresponds to the test PR and open.
+
+    Args:
+        response_json (json): info about PR
+
+    Returns:
+         (json): {'user': <username>, 'url': <pull_req_url>}
+    """
+    try:
+        if response_json['total_requests'] > 0:
+            for request in response_json['requests']:
+                if ('Add CI tests' or 'test' in request['title']) and (request['status'] == 'Open'):
+                    pull_req_url = DIST_GIT_URL + request['project']['url_path'] + '/pull-requests'
+                    return {'user': request['user'], 'url': pull_req_url}
+
+    except Exception:
+        # {'error': 'Project not found', 'error_code': 'ENOPROJECT'}
+        return
+
+
+def get_all_projects(site_url):
+    """Get all projects from site using API
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+
+    Returns:
+        response (json): all projects
+    """
+    site = site_url + 'api/0/projects'
+
+    response = requests.get(site)
+    return response.json()
+
+
+def get_projects_names(json_response):
+    """Get projects names from the all projects
+
+    Args:
+        json_response (json): all projects
+
+    Returns:
+        projects_names (list of strings) names of projects
+    """
+
+    projects_names = []
+
+    for project in json_response['projects']:
+        project_name = project['fullname']
+        if 'forks/' not in project_name:
+            projects_names.append(project_name)
+
+    return projects_names
+
+
+def get_projects_url_patches(json_response):
+    """Get projects url patches"""
+
+    projects_url_patches = []
+
+    for project in json_response['projects']:
+        project_name = project['url_path']
+
+        projects_url_patches.append(project_name)
+
+    return projects_url_patches
+
+
+def get_test_yaml_from_site(site_url, package, test_file='tests.yml'):
+    """Get test.yaml from the site
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+        test_file (string): file to get from site.
+
+    Returns:
+        test.yaml (raw string)
+    """
+
+    if 'upstreamfirst' in site_url:
+        test_file_url = site_url + package + '/raw/master/f/' + test_file
+    elif 'fedoraproject' in site_url:
+        test_file_url = site_url + '/rpms/' + package + '/raw/master/f/tests/' + test_file
+    else:
+        return
+
+    response = requests.get(test_file_url)
+    return response.text
+
+
+def get_url_to_test_yml(site_url, package):
+    """Get url to the test.yml file
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+
+    Returns:
+        url (string)
+    """
+    if 'upstreamfirst' in site_url:
+        test_file_url = site_url + package + '/blob/master/f/tests.yml'
+    elif 'fedoraproject' in site_url:
+        test_file_url = site_url + 'rpms/' + package + '/blob/master/f/tests/tests.yml'
+    else:
+        return
+
+    return test_file_url
+
+
+def check_if_test_yml_exists(test_file_url):
+    """Checks if tesy.yml exists
+
+    Args:
+        test_file_url (string): url of the site
+
+    Returns:
+        True/False (bool) if response.status_code
+    """
+    response = requests.get(test_file_url)
+    if response.status_code == 200:
+        return True
+    else:
+        return False
+
+
+def get_test_tags(raw_text):
+    """Just returns existed test-tags
+
+    Args:
+        raw_text (string): raw text output from the requests
+
+    Returns:
+        tags (list if strings)
+    """
+    test_tags = []
+
+    for tag in ['classic', 'container', 'atomic']:
+        if tag in raw_text:
+            test_tags.append(tag)
+
+    return test_tags
+
+
+def handle_test_tags(site_url, package):
+    """Function grts new patch to the test.yaml if existing test.yaml includes test file
+
+    Args:
+        site_url (string): url of the site
+            for example: 'https://upstreamfirst.fedorainfracloud.org/'
+        package (string): name of the package
+
+    Returns:
+        tags (list of strings)
+    """
+
+    raw_text = get_test_yaml_from_site(site_url, package)
+
+    if 'Page not found' in raw_text:
+        return []
+
+    test_tags = get_test_tags(raw_text)
+
+    if not test_tags:
+        new_test_file = re.findall(r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', raw_text)
+
+        if new_test_file:
+            raw_text = get_test_yaml_from_site(site_url, package, new_test_file[-1])
+            test_tags = get_test_tags(raw_text)
+
+    return test_tags
+
+
+def test_tags_to_dict(test_tags):
+    """Convert test-tags list to the dictionary"""
+
+    tags_dict = {'classic': False, 'container': False, 'atomic': False}
+    try:
+        for tag in test_tags:
+            tags_dict[tag] = True
+
+    except Exception:
+        pass
+    return tags_dict
+
+
+def get_list_of_packages_from_the_file(file):
+    """Reads list of packages from a file
+
+    Args:
+        file (file): file to get packages from
+
+    Returns:
+        packages (list of packages)
+    """
+    with open(file) as packages_file:
+        return packages_file.read().splitlines()
+
+
+def write_results_to_the_file(result, filename='results.txt'):
+    """Write results to the file
+
+    Args:
+        result (json): package's info
+        filename (string): name of the file to write packages
+
+    Returns:
+        packages (list of packages)
+    """
+    with open(filename, 'a') as my_file:
+        my_file.write('{}\n'.format(result))
+
+    my_file.close()
+    return filename
+
+
+def delete_files_before_running(files_to_delete):
+    """Function deletes (cleanup) files with data
+
+    Args:
+        files_to_delete (list): list of files to delete
+
+    Returns:
+        (None) if everything is alright
+    """
+    for file_name in files_to_delete:
+        try:
+            os.remove(file_name)
+        except OSError:
+            pass
+
+
+if __name__ == '__main__':
+    exit(0)

