Quelle  comm-task-env

#!/usr/bin/python3 -u
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Thunderbird build environment prep for run-task,
for use with comm-central derived repositories.

This script is meant to run prior to run-task on repositories like
comm-central that need to check out a copy of a mozilla repository
in order to build.
See bug 1491371 for background on why this is necessary.

A project will have a file named ".gecko_rev.yml" in it's root. See the
constant "GECKO_REV_CONF" if you want to change that. To download it, the
script uses the project repository URL and the revision number.
Those are defined in the environment variables:
COMM_HEAD_REPOSITORY
COMM_HEAD_REV

.gecko_rev.yml has a structure like (for comm-central):
```
GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central
GECKO_HEAD_REF: default
```
or for branches:
```
GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified
GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta
GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH
GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910

Note about GECKO_HEAD_REV and GECKO_HEAD_REF:
GECKO_HEAD_REF is a branch name or "default".
GECKO_HEAD_REV is a revision hash.
```
"""

import sys

import os
import socket
import time
from datetime import datetime
from pprint import pformat

import urllib.error
import urllib.request

import yaml

if sys.version_info[0:2] < (3, 5):
    print('run-task-wrapper requires Python 3.5+')
    sys.exit(1)

GECKO_REV_CONF = ".gecko_rev.yml"
DEBUG = bool(os.environ.get("RTW_DEBUG", False))


def print_message(msg, prefix=__file__, level=""):
    """
    Print messages.
    :param object msg: message to print, usually a string, but not always
    :param str prefix: message prefix
    :param str level: message level (DEBUG, ERROR, INFO)
    """
    if not isinstance(msg, str):
        msg = pformat(msg)
    now = datetime.utcnow().isoformat()
    # slice microseconds to 3 decimals.
    now = now[:-3] if now[-7:-6] == '.' else now
    if level:
        sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format(
            prefix=prefix, now=now, level=level, msg=msg))
    else:
        sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format(
            prefix=prefix, now=now, msg=msg))
    sys.stdout.flush()


def error_exit(msg):
    """Print the error message and exit with error."""
    print_message(msg, level="ERROR")
    if DEBUG:
        raise Exception(msg)

    sys.exit(1)


def print_debug(msg):
    """Prints a message with DEBUG prefix if DEBUG is enabled
    with the environment variable "RTW_DEBUG".
    """
    if DEBUG:
        print_message(msg, level="DEBUG")


def check_environ():
    """Check that the necessary environment variables to find the
    comm- repository are defined. (Set in .taskcluster.yml)
    :return: tuple(str, str)
    """
    print_debug("Checking environment variables...")
    project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None)
    project_head_rev = os.environ.get("COMM_HEAD_REV", None)

    if project_head_repo is None or project_head_rev is None:
        error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format(
            project_head_repo, project_head_rev)

    print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format(
        project_head_repo, project_head_rev))
    return project_head_repo, project_head_rev


def download_url(url, retry=1):
    """Downloads the given URL. Naively retries (when asked) upon failure
    :param url: str
    :param retry: int
    :return: str
    """
    # Use 1-based counting for display and calculation purposes.
    for i in range(1, retry+1):
        try:
            print_message('Fetching {}. Attempt {} of {}.'.format(
                url, i, retry))
            with urllib.request.urlopen(url, timeout=10) as response:
                data = response.read().decode("utf-8")
            return data
        except (urllib.error.URLError, socket.timeout) as exc:
            print_message('Unable to retrieve {}'.format(url))
            if isinstance(exc, urllib.error.URLError):
                print_message(exc.reason)
            else:  # socket.timeout
                print_message('Connection timed out.')

            if i < retry:  # No more retries
                wait_time = i * 5  # fail #1: sleep 5s. #2, sleep 10s
                print_message('Retrying in {} seconds.'.format(wait_time))
                time.sleep(wait_time)

    error_exit('No more retry attempts! Aborting.')


def fetch_gecko_conf(project_head_repo, project_revision):
    """Downloads .gecko_rev.yml from the project repository
    :param project_head_repo: str
    :param project_revision: str
    :return: dict
    """
    gecko_conf_url = '/'.join(
        [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF])

    gecko_conf_yml = download_url(gecko_conf_url, retry=5)

    try:
        gecko_conf = yaml.safe_load(gecko_conf_yml)
        return gecko_conf
    except yaml.YAMLError as exc:
        err_txt = ["Error processing Gecko YAML configuration."]
        if hasattr(exc, "problem_mark"):
            mark = exc.problem_mark  # pylint: disable=no-member
            err_txt.append("Error position: line {}, column {}".format(
                mark.line + 1, mark.column + 1))
        error_exit('\n'.join(err_txt))


def update_environment(gecko_conf):
    """Adds the new variables defined in gecko_conf to the
    running environment.
    :param gecko_conf: dict
    """
    print_message("Updating environment with:")
    print_message(gecko_conf)
    os.environ.update(gecko_conf)

    print_debug("New environment:")
    print_debug(os.environ)


def exec_run_task(args):
    """Executes run-task with a modified environment."""
    print_message("Executing: {}".format(pformat(args)))
    os.execv(args[0], args[0:])


def main():
    """Main function."""
    args = sys.argv[1:]  # Remaining args starting with run-task

    project_head_repo, project_revision = check_environ()
    gecko_conf = fetch_gecko_conf(project_head_repo, project_revision)
    update_environment(gecko_conf)
    exec_run_task(args)


if __name__ == "__main__":
    main()