# -*- coding: utf-8 -*-
#
# Copyright © 2012,2013 Pôle de compétences EOLE <eole@ac-dijon.fr>
#
# License CeCILL:
#  * in french: http://www.cecill.info/licences/Licence_CeCILL_V2-fr.html
#  * in english http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
##########################################################################

""" Package management with multiple package manager support

This class allows to manage package installation and removing,
system package update.

    Package management interface supports :
       - apt

    To be done :
       - yum
       - pacman
       - urpmi
"""
import os
import sys
import argparse
import pty
import json
import logging
try:
    import urlparse
except Exception:
    import urllib.parse as urlparse
import tempfile
from shutil import rmtree
from time import time
import subprocess
from contextlib import contextmanager
from pyeole import scriptargs

from pyeole.log import init_logging
from pyeole.process import creole_system_out, creole_system_code, \
    system_progress_out
from pyeole.process import system_code, system_out
from pyeole.diagnose import test_http, test_tcp
from pyeole.encode import normalize
from pyeole.translation.i18n import _
from pyeole.diagnose.diagnose import REPORT_MAJ_FILE, MAJ_SUCCES_LOCK

from creole.client import CreoleClient, NotFoundError, \
    TimeoutCreoleClientError, CreoleClientError
from creole.config import templatedir
from creole.config import distrib_dir

from creole.eoleversion import UBUNTU_VERSION
from creole.eoleversion import EOLE_VERSION
from creole.eoleversion import EOLE_RELEASE
from creole.eoleversion import ENVOLE_VERSION

if not CreoleClient.is_in_lxc():
    from creole.containers import is_lxc_running
    from creole.error import VirtError
    from creole.template import CreoleTemplateEngine

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

try:
    import apt
    import apt_pkg
    from aptsources.sourceslist import SourceEntry
except ImportError as err:
    log.debug(err)

try:
    import yum
except ImportError:
    log.debug(_('python-yum not found'))


_TESTRUN = False

_EOLE_APT_CONF_FILE = '/etc/apt/apt-eole.conf'

_SOURCES_LISTS_DIR = '/etc/apt/sources.list.d/'
_APT_CONFIG_DIR = '/etc/apt/apt.conf.d/'
_APT_PROXY_FILE = os.path.join(_APT_CONFIG_DIR, '02eoleproxy')

_EOLEHUB_COMMON_ENV = '/etc/eole/containers.conf.d/common.env'

_APT_TERM_FILE = '/var/log/apt/term.log'
"""Default log file for apt-get standard output
"""

_CONFFILE_DEFAULT_ANSWER = 'y'
"""Default answer if progress installer get an question.
"""

_DEBIAN_FRONTEND = 'noninteractive'
"""Make dpkg and debconf completely silent
"""

_APT_PROXY_OPT = "Acquire::http::Proxy"

_APT_CACHER_ERROR_MSG = _('cannot connect to apt-cacher-ng, please check log in /var/log/apt-cacher-ng/ and restart service with "systemctl start apt-cacher-ng" command')
_APT_CACHER_ERROR_MSG_CONTAINER = _('cannot connect to apt-cacher-ng inside container, please check log in /var/log/apt-cacher-ng/ and restart service with "systemctl restart apt-cacher-ng" command')


_SOURCES_CONF = {'ubuntu': {'mirror': 'http://eole.ac-dijon.fr/ubuntu',
                             'sources_file': '/etc/apt/sources.list',
                             'level': {'proposed': ['stable', 'security', 'updates'],
                                        'updates': ['stable', 'security'],
                                        'security': ['stable']}},
                 'eole': {'mirror': 'http://eole.ac-dijon.fr/eole',
                           'sources_file': '/etc/apt/sources.list.d/eole.list',
                           'level': {'proposed-updates': ['stable', 'security', 'updates'],
                                      'updates': ['stable', 'security'],
                                      'security': ['stable']}},
                }
_MIRROR_DIST = {'Ubuntu': '{0}/main/binary-amd64'.format(UBUNTU_VERSION),
                'EOLE': 'eole-{0}-unstable'.format(EOLE_VERSION),
                'Envole': 'envole-{0}-unstable'.format(ENVOLE_VERSION)}

_ARCHIVE_DIR = '/var/cache/apt/archives/'
_LISTS_DIR = '/var/lib/apt/lists'

#valeurs par défaut si impossible de récupérer celles configurées
_EOLE_MIRRORS = ['eole.ac-dijon.fr', 'ftp.crihan.fr']
_ENVOLE_MIRRORS = ['eole.ac-dijon.fr', 'ftp.crihan.fr']
_UBUNTU_MIRRORS = ['eole.ac-dijon.fr', 'ftp.crihan.fr']


DEFAULT_DIST = 'eole-{0}'.format(EOLE_RELEASE)
DEFAULT_LEVEL = 'updates'

#exceptions ignorées avec l'option "-i"
CREOLE_ERRORS = (NotFoundError, TimeoutCreoleClientError, CreoleClientError)


class PackageError(Exception):
    """Base exception for package management

    """
    pass

class PackageNotFoundError(PackageError):
    """Requested package is not found

    """
    pass


class RepositoryError(PackageError):
    """Unable to reach APT repository

    """
    pass

class AptError(PackageError):
    """Error running apt-get
    """
    pass

class AptProxyError(AptError):
    """Error connecting to proxy
    """
    pass

class AptCacherError(AptError):
    """Error connecting to proxy
    """
    pass


@contextmanager
def tmp_engine_context(engine, context):
    old_state = {key: engine.creole_variables_dict.get(key) for key in context
                 if key in engine.creole_variables_dict}
    keys = context.keys()
    engine.creole_variables_dict.update(context)
    try:
        yield engine
    finally:
        for key in keys:
            del engine.creole_variables_dict[key]
        engine.creole_variables_dict.update(old_state)


def parse_apt_cmdline():
    """Parse commande line.
    """
    ubuntu_components = ['main', 'universe', 'multiverse', 'restricted']

    configure_args = ['mirror', 'vendor', 'dist', 'level', 'component']
    pkg_args = ['packages']

    eole_components = ['main', 'cloud']

    # Global parser
    parser = argparse.ArgumentParser(description=_('Run apt in non-interactive'),
                                     parents=[scriptargs.container(),
                                              scriptargs.logging('info')])

    parser.add_argument('-j', '--json', action='store_true',
                        help=_('Output in JSON format'))

    parser.add_argument('-o', '--only-install', action='store_true',
                        default=False,
                        help=_('install does not update existing packages'))

    parser.add_argument('-s', '--simulate', '--just-print',
                        '--dry-run', '--recon', '--no-act',
                        action='store_true',
                        help=_('No action; perform a simulation'))

    parser.add_argument('-z', '--silent', action='store_true',
                        default=False, help=_('no lines are printed'))

    # per action subparser
    actions = parser.add_subparsers(dest='action',
                                    title=_('APT actions'),
                                    help=_('actions help'))

    # Configuration of sources.list
    configure = actions.add_parser('configure',
                                   help=_('configure sources.list'))
    vendor = configure.add_subparsers(dest='vendor',
                                      title=_('Vendor sources.list configuration'),
                                      help=_('distribution vendor'))

    ubuntu = vendor.add_parser('ubuntu', help=_('Ubuntu sources.list'))
    ubuntu.add_argument('-m', '--mirror', metavar='URI', type=str,
                        default='http://eoleng.ac-dijon.fr/ubuntu',
                        help=_('URI of the mirror'))
    ubuntu.add_argument('-d', '--dist', type=str,
                        default=UBUNTU_VERSION, help=_('distribution name'))
    ubuntu.add_argument('-l', '--level',
                        choices=_SOURCES_CONF['ubuntu']['level'].keys(),
                        default='updates',
                        help=_('update level'))
    ubuntu.add_argument('-c', '--component', nargs='+',
                        choices=ubuntu_components,
                        default=ubuntu_components,
                        help=_('component to activate'))
    ubuntu.set_defaults(sources_file='/etc/apt/sources.list')
    ubuntu.set_defaults(func='configure')
    ubuntu.set_defaults(func_args=configure_args)

    eole = vendor.add_parser('eole', help=_('EOLE sources.list'))
    eole.add_argument('-m', '--mirror', metavar='URI', type=str,
                      default='http://eole.ac-dijon.fr/eole',
                      help=_('URI of the mirror'))

    eole.add_argument('-d', '--dist', type=str,
                      choices=['stable', 'testing', 'unstable'],
                      default='stable', help=_('distribution name'))

    eole.add_argument('-c', '--component', nargs='+',
                      choices=eole_components,
                      default=eole_components,
                      help=_('component to activate'))
    eole.add_argument('-l', '--level',
                      choices=('stable', 'updates'),
                      default='updates',
                      help=_('update level'))

    eole.set_defaults(sources_file='/etc/apt/sources.list.d/eole.list')
    eole.set_defaults(func='configure')
    eole.set_defaults(func_args=configure_args)
    update = actions.add_parser('update', help=_('update list of indexes'))
    update.set_defaults(func='update')

    upgrade = actions.add_parser('upgrade', help=_('upgrade packages'))
    upgrade.set_defaults(func='upgrade')

    down_upgrade = actions.add_parser('download-upgrade', help=_('download upgrade distribution'))
    down_upgrade.set_defaults(func='download_upgrade')

    dist_upgrade = actions.add_parser('dist-upgrade', help=_('upgrade distribution'))
    dist_upgrade.set_defaults(func='dist_upgrade')

    list_upgrade = actions.add_parser('list-upgrade', help=_('list packages to upgrade'))
    list_upgrade.set_defaults(func='list_upgrade')

    fetch_archives = actions.add_parser('fetch_archives', help=_('download package'))
    fetch_archives.add_argument('packages', metavar='packages', type=str, nargs='+',
                                help=_('package names'))
    fetch_archives.set_defaults(func='fetch_archives')
    fetch_archives.set_defaults(func_args=pkg_args)

    install = actions.add_parser('install', help=_('install packages'))
    install.add_argument('packages', metavar='package', type=str, nargs='+',
                         help=_('package names'))
    install.set_defaults(func='install')
    install.set_defaults(func_args=pkg_args)

    remove = actions.add_parser('remove', help=_('remove packages'))
    remove.add_argument('packages', metavar='package', type=str, nargs='+',
                        help=_('package names'))
    remove.set_defaults(func='remove')
    remove.set_defaults(func_args=pkg_args)

    check = actions.add_parser('check', help=_('verify dpkg status'))
    check.set_defaults(func='check')

    opts = parser.parse_args()

    if opts.action == 'configure' and opts.vendor == 'eole':
        opts = _prepare_eole_configure(opts, parser)

    if opts.verbose:
        opts.log_level = 'info'
    if opts.debug:
        opts.log_level = 'debug'
    if opts.silent:
        opts.log_level = 'error'
    if opts.json:
        opts.log_level = 'critical'

    return opts


def _prepare_eole_configure(options, parser):
    """Prepare a configure for EOLE

    Unstable and testing do not have an update “level”, use “stable”
    to strip it.

    """
    if options.dist == 'stable':
        options.dist = 'eole-{0}'.format(EOLE_RELEASE)
        options.level = 'updates'
    elif options.dist == 'testing':
        options.dist = 'eole-{0}-testing'.format(EOLE_VERSION)
        options.level = 'stable'
    elif options.dist == 'unstable':
        options.dist = 'eole-{0}-unstable'.format(EOLE_VERSION)
        options.level = 'stable'

    return options


def _retrieve_dist_from_kwargs(dist):
    dist = dist.split('-')
    if len(dist) == 1:
        return dist[0]
    if len(dist) == 2:
        return 'stable'
    else:
        return dist[2]


def _build_apt_cmd_line(**kwargs):
    """Build APT commande line from arguments

    This is usefull to pass apt-eole calls in containers.

    """
    action = kwargs['action']
    silent = kwargs['silent']
    cmd = ['apt-eole']
# useless with system_progress_out() call
#    if silent is True:
#        cmd.extend(['-z'])
    only_install = kwargs.get('only_install', True)
    if only_install is True:
        cmd.extend(['-o'])
    if kwargs.get('container', None):
        cmd.extend(['--container', kwargs.get('container')])
    if kwargs.get('json', False):
        cmd.append('--json')
    if kwargs.get('simulate', False):
        cmd.append('--simulate')
    cmd.extend(['--log-level', kwargs.get('log_level', 'info')])
    cmd.append(action)

    if action == 'configure':
        cmd.append(kwargs['vendor'])
        mirror = kwargs['mirror']
        if mirror is not None:
            cmd.extend(['--mirror', mirror])
        level = kwargs['level']
        if level is not None:
            cmd.extend(['--level', level])
        dist = kwargs['dist']
        if dist is not None:
            cmd.extend(['--dist', _retrieve_dist_from_kwargs(dist)])
        if kwargs.get('component', None):
            cmd.extend(['--component'] + kwargs['component'])
        # cmd.extend(['--source-file', kwargs['sources_files']])
    elif action == 'install':
        #force l'installation du paquet eole-lxc-container-pkg dans les conteneurs #8675
        cmd.extend(['eole-lxc-container-pkg'] + kwargs['packages'])
    elif action == 'remove':
        cmd.extend(kwargs['packages'])
    elif action == 'fetch_archives':
        cmd.extend(kwargs['packages'])

    return cmd


def _run_apt_get(arg_list, silent=False, display_info=None):
    """Call apt-get command

    """
    cmd = ['apt-get', '-c', _EOLE_APT_CONF_FILE]
    cmd.extend(arg_list)

    # Force non-interactive in postinst
    os.environ["DEBIAN_FRONTEND"] = _DEBIAN_FRONTEND
    # rt ticket #49720
    if "PATH" not in os.environ or "sbin" not in os.environ["PATH"]:
        os.environ["PATH"] = "/usr/sbin:/usr/bin:/sbin:/bin"

    log.debug('Run: {0}'.format(' '.join(cmd)))
    if silent:
        msg = _('Running {0} for root').format(arg_list[0])
        code, stdout, stderr = system_progress_out(cmd, msg)
    else:
        proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
        proc.wait()
        code = proc.returncode
        if code != 0:
            stderr = proc.stderr.read()
    if code != 0:
        if isinstance(stderr, bytes):
            stderr = stderr.decode('UTF-8')
        # ticket #17482 make error message great again
        unauthenticated_msg = ["E: There were unauthenticated packages and -y was used without --allow-unauthenticated",
                "E: Il y a eu des paquets non authentifiés et -y a été employé sans --allow-unauthenticated"]
        if stderr.strip() in unauthenticated_msg:
            stderr = _(u"There were unauthenticated packages and Maj-Auto has prevented their installation")
        elif "Connexion à localhost: 3142 (127.0.0.1) impossible." in stderr.strip():
            stderr = stderr+"\n"+_APT_CACHER_ERROR_MSG
        elif "Impossible de se connecter à 192.0.2.1:3142" in stderr.strip():
            stderr = stderr+"\n"+_APT_CACHER_ERROR_MSG_CONTAINER
        raise AptError(_(u"apt-get ({0}) returned an error (code {1}). See {2}\n{3}")\
                .format(u" ".join(cmd[1:]), code, _APT_TERM_FILE, stderr))
    return code

def apt_source_get_uri(line):
    """Get URI from a source line

    :param line: The line from a source list to parse
    :return: The URI string or '' if the line is ill-formed
    """
    return SourceEntry(line).uri

def apt_cli():
    """Get command line arguments and run the action

    """
    opts = vars(parse_apt_cmdline())
    if opts['log_level'] == 'info':
        tmpl_loglevel = 'warning'
    else:
        tmpl_loglevel = opts['log_level']
    creole_template_log = init_logging(name='creole.template',
                                       level=tmpl_loglevel)

    cli_log = init_logging(name='apt-eole', level=opts['log_level'],
                           as_root=True)

    try:
        only_install = opts.get('only_install', False)
        pkgmgr = EolePkg('apt', only_install=only_install)
        if opts.get('simulate', False):
            pkgmgr.set_option('APT::Get::Simulate', 'true')
        if opts.get('json', False):
            pkgmgr.set_json_output(root=True)
        func_args = {}
        for arg_name in opts.get('func_args', []):
            if opts.get(arg_name, None) is None:
                continue
            func_args[arg_name] = opts[arg_name]
        if opts.get('simulate', False):
            func_args['simulate'] = opts['simulate']

        getattr(pkgmgr, opts['func'])(container=opts['container'],
                                       silent=opts['silent'],
                                       **func_args)
    except Exception as err:
        if opts['debug']:
            cli_log.error(err, exc_info=True)
        else:
            cli_log.error(err)
        sys.exit(1)
    sys.exit(0)


def _sort_containers(containers):
    """Sort a containers list
    :param containers: containers list
    :type containers: `list`
    :return: containers list sorted
    :rtype: `list`
    """
    containers = sorted(containers)
    if 'root' in containers:
        # 'root' should be the first one
        containers.remove('root')
        containers.insert(0, 'root')
    return containers


def report(retval, errmsg=None, reconf=False):
    """
    retval: 0->OK, 1->NOK, 2->IN PROGRESS, 3->ALREADY UPDATE
    """
    if retval == 0:
        msg = _('Update completed successfully')
        if not reconf:
            open(MAJ_SUCCES_LOCK, 'w').write('')
    elif retval == 1:
        msg = _('Error updating : {0}').format(errmsg)
    elif retval == 2:
        msg = _('Update in progress')
    elif retval == 3:
        retval = 0
        msg = _('Server already updated')
    fh = open(REPORT_MAJ_FILE, 'w')
    fh.write(u"# -*- coding: UTF-8 -*-\n")
    fh.write('DATE="{0}"\n'.format(int(time())))
    fh.write('STATUS="{0}"\n'.format(retval))
    fh.write('MSG="{0}"\n'.format(msg))
    fh.close()


class NonInteractiveInstallProgress(apt.progress.base.InstallProgress):
    """Make installation/remove process silent.
    """

    def __init__(self, silent=False):
        """Initialize the NonInteractiveInstallProgress
        """
        self.silent = silent
        apt.progress.base.InstallProgress.__init__(self)

    def conffile(self, current, new):
        os.write(self.master_fd, "%s\n" % _CONFFILE_DEFAULT_ANSWER)

    def fork(self):
        (pid, self.master_fd) = pty.fork()
        if pid == 0:
            os.environ["DEBIAN_FRONTEND"] = _DEBIAN_FRONTEND
            # rt ticket #49720
            if "PATH" not in os.environ or "sbin" not in os.environ["PATH"]:
                os.environ["PATH"] = "/usr/sbin:/usr/bin:/sbin:/bin"
        return pid

    def update_interface(self):
        try:
            if self.silent:
                # Avoid blocking file descriptor
                os.read(self.master_fd, 512),
            else:
                print (os.read(self.master_fd, 512))
        except:
            pass
        apt.progress.base.InstallProgress.update_interface(self)


class EolePkgApt(object):
    """apt wrapper class.
    This class mangage the interaction with "apt" based systems, like debian or ubutnu.
    This is a 'private' class it is made to be used by :class:`EolePkg` with the 'apt'
    parameter

    - prototype: ``EolePkgApt(root)`` with ``root`` the root directory for
    package installation

    When :method:`__init__` is called you initialize a new interface for package
    management.

    """
    def __init__(self, root=None, test=False, container_mode=None, only_install=True, ignore=False):
        """Initialize the apt wrapper
        Load end prepare all the necessary to manage packages with `apt`

        :param root: The system directory to be the root for package installation
        :type root: directory `string`
        :raise ImportError: if python-apt is not available.
        """
        self.cache = None
        self._rootdir = root
        self._only_install = only_install
        self._read_config_file()
        # Progress
        self._acquire_progress = apt.progress.text.AcquireProgress()
        self._install_progress = NonInteractiveInstallProgress()
        # silent version
        self._silent_install_progress = NonInteractiveInstallProgress(silent=True)

        self.client = CreoleClient()

        # Do not fail on containers if not instanciated
        self._instantiated = None

        self._apt_cacher_port = None

        # Do not run in container mode within a container
        if container_mode is not None:
            self._container_mode = container_mode
        else:
            self._container_mode = None

        self._return_json = False

        self.set_test_run(_TESTRUN)
        self.set_test_run(test)
        self.groups = None
        self.ignore = ignore

    def is_container_mode(self):
        if self._container_mode is None:
            try:
                self._container_mode = not self.client.is_in_lxc()\
                                      and self.client.get_creole('mode_conteneur_actif') == 'oui'
                if self.ignore:
                    self.get_groups()
            except CREOLE_ERRORS as err:
                if self.ignore:
                    self._container_mode = None
                    return False
                else:
                    raise err
        return self._container_mode

    def is_instantiated(self):
        if self._instantiated is None:
            self._instantiated = self.client.get_creole('module_instancie') == 'oui'
        return self._instantiated

    def get_apt_cacher_port(self):
        if self._apt_cacher_port is None:
            try:
                self._apt_cacher_port = self.client.get_creole('apt_cacher_port', None)
            except CREOLE_ERRORS as err:
                if self.ignore:
                    self._apt_cacher_port = None
                else:
                    raise err
        return self._apt_cacher_port

    def set_test_run(self, value):
        """Set a value to :attr:`self._testrun`
        Used by the unit test for dry run, don't use this
        """
        self._testrun = value

    def get_groups(self):
        if self.groups is None:
            groups = {'group_infos': {}}
            groups['groups'] = self.client.get_groups()
            for group_name in groups['groups']:
                groups['group_infos'][group_name] = self.client.get_group_infos(group_name)
            if self.is_container_mode():
                # add "group_info" for basic containers too (#12964)
                for group in self.client.get_containers():
                    if group['container_group'] != 'all' and group['name'] not in groups['group_infos']:
                        groups['group_infos'][group['name']] = groups['group_infos'][group['container_group']]
            self.groups = groups
        return self.groups


    def _load_apt_cache(self, action=None):
        """Load APT cache

        :param action: which action was called
        :type action: `str`

        """
        if self.cache is None:
            try:
                self.cache = apt.Cache(rootdir=self._rootdir)
            except SystemError as err:
                if action != 'configure':
                    raise err
            else:
                #Test dpkg
                if self.cache.dpkg_journal_dirty:
                    raise SystemError(_('dpkg journal is dirty, please run '
                                        '"dpkg --configure -a" and retry it'))
                self.cache.open()


    def _run(self, action, func, container=None, silent=False, **kwargs):
        """Run apt action.

        :param silent_apt: avoid apt making outputs on standard output
        :type silent_apt: `bool`

        """
        if self.is_container_mode() and container not in [None, 'all', 'root', 'current']:
            return {}

        return {'root': getattr(self, func)(silent=silent, **kwargs)}


    def _fast_update(self, silent, ctx, **kwargs):
        """"
        rsync des fichiers du maître
        au lieu d' "apt-get update" dans les conteneurs
        """
        msg = _('Running {0} for container {1}').format('update', ctx['name'])
        container_sources = os.path.join(ctx['path'], _SOURCES_LISTS_DIR[1:])
        cmd = ['diff', _SOURCES_LISTS_DIR, container_sources]
        log.debug('Run: {0}'.format(' '.join(cmd)))
        diff = creole_system_out(cmd)
        if diff[0] != 0:
            # FIXME: warn or info ?
            c_name = ctx['name']
            log.warning(_('Container {0} has specific sources.list').format(c_name))
            cmd = ['apt-eole', 'update']
            log.debug('Run: {0} ({1})'.format(' '.join(cmd), c_name))
            code, stdout, stderr = system_progress_out(cmd, container=ctx,
                                                       message=msg)
        else:
            dirname = os.path.dirname(_LISTS_DIR)[1:]
            cpath = os.path.join(ctx['path'], dirname)
            cmd = ['/usr/bin/rsync', '-a', _LISTS_DIR, cpath]
            log.debug('Run: {0}'.format(' '.join(cmd)))
            code, stdout, stderr = system_progress_out(cmd, message=msg)
        if code != 0:
            cmd_string = ' '.join(cmd)
            msg = _(u"Can not execute '{0}'")
            if stderr:
                msg += u": {1}"
            raise SystemError(msg.format(cmd_string, stderr))


    def _run_containers(self, action, func, container, silent=False, **kwargs):
        """Run apt-eole in containers
        """
        fast_func = '_fast_{0}'.format(func)
        outputs = {}
        if fast_func in dir(self):
            if self.is_container_mode() and container != 'current':
                groups = self.get_groups()
                no_container = False
                if container == 'all':
                    containers = groups['groups']
                elif container not in [None, 'root']:
                    containers = [container]
                else:
                    no_container = True

                if not no_container:
                    containers = [name for name in containers if name not in ['root', 'all']]
                    for container in containers:
                        container_infos = groups['group_infos'][container]
                        if not is_lxc_running(container_infos):
                            msg = _('Container {0} is not running.').format(container)

                            if self.is_instantiated():
                                raise VirtError(msg)
                            else:
                                log.warning(msg)
                                continue

                        outputs[container] = getattr(self, fast_func)(silent=silent, ctx=container_infos, **kwargs)
        else:
            if self.is_container_mode() and container != 'current':
                groups = self.get_groups()
                no_container = False
                if container == 'all':
                    containers = groups['groups']
                elif container not in [None, 'root']:
                    containers = [container]
                else:
                    no_container = True

                if not no_container:
                    # Strip unwanted containers
                    containers = [name for name in containers if name not in ['root', 'all']]
                    cmd = _build_apt_cmd_line(action=action,
                                              container='current',
                                              json=self._return_json,
                                              silent=silent,
                                              only_install=True,
                                              **kwargs)
                    #log.debug('Run: {0}'.format(' '.join(cmd)))
                    os.putenv('DEBIAN_FRONTEND', _DEBIAN_FRONTEND)
                    for container in containers:
                        try:
                            container_infos = groups['group_infos'][container]
                        except KeyError as err:
                            raise ValueError(_(u"Unknown container: {}").format(container))

                        if not is_lxc_running(container_infos):
                            msg = _('Container {0} is not running.').format(container)

                            if self.is_instantiated():
                                raise VirtError(msg)
                            else:
                                log.warning(msg)
                                continue
                        msg = _('Running {0} for container {1}').format(action, container)
                        log.debug('Run: {0} ({1})'.format(' '.join(cmd), container))
                        if self._return_json or silent:
                            code, stdout, stderr = system_progress_out(cmd, container=groups['group_infos'][container], message=msg)
                        else:
                            log.info(msg)
                            code = creole_system_code(cmd, container=groups['group_infos'][container])
                            stdout = None
                            stderr = None
                        if code == 0:
                            if self._return_json:
                                # Every container see itself as root
                                try:
                                    outputs[container] = json.loads(stdout)['root']
                                except ValueError:
                                    cmd_string = ' '.join(cmd)
                                    msg = _('Parsing error while executing \'{0}\' for container \'{1}\' : {2}')
                                    if stderr:
                                        msg += u", {3}"
                                    raise SystemError(msg.format(cmd_string, container, stdout, stderr))
                            else:
                                outputs[container] = stdout
                        else:
                            cmd_string = ' '.join(cmd)
                            msg = _(u"Can not execute '{0}' for container '{1}'")
                            #print stdout, stderr
                            if stderr:
                                msg += u": {2}"
                            raise SystemError(msg.format(cmd_string, container, stderr))
        return outputs


    def _read_config_file(self):
        """Read the EOLE configuration file.

        """
        if os.access(_EOLE_APT_CONF_FILE, os.R_OK):
            try:
                apt_pkg.read_config_file(apt_pkg.config, _EOLE_APT_CONF_FILE)
            except SystemError as err:
                log.error(_('Unable to load EOLE apt configuration: {0}').format(normalize(str(err))))
                log.debug(err, exc_info=True)


    def set_option(self, option, value):
        """Set APT option

        """
        apt_pkg.config.set(option, value)


    def get_package(self, pkgname):
        """Search a package in package database.

        :param pkgname: The package name
        :type pkgname: `str`
        :return: a package from the package database.
        :rtype: `apt.Package`

        """
        try:
            self._load_apt_cache()
            return self.cache[pkgname]
        except KeyError:
            raise PackageNotFoundError(_('Package {0} not found').format(pkgname))

    def get_rev_depends(self, pkgname):
        """ get reverse depends list

        :param pkgname: The package name
        :type pkgname: `str`
        :return: a package name list.
        :rtype: `list`
        """
        dep_list = []
        self._load_apt_cache()
        pkg = self.cache[pkgname]
        for el in self.cache.keys():
            pkg = self.cache[el]
            deps = pkg.candidateDependencies
            for dep in deps:
                for el in dep:
                    if pkgname == el.name:
                        dep_list.append(pkg.name)
        return dep_list

    @staticmethod
    def _get_proxy(mirror=None):
        env_proxy = os.environ.get('http_proxy')
        proxy = None
        if env_proxy:
            proxy = env_proxy
        else:
            if mirror is not None:
                host = urlparse.urlparse(mirror).netloc
                host_proxy = apt_pkg.config.get("{0}::{1}".format(_APT_PROXY_OPT, host))
            else:
                host_proxy = False
            if host_proxy:
                proxy = host_proxy
            else:
                default_proxy = apt_pkg.config.get(_APT_PROXY_OPT)
                if default_proxy:
                    proxy = default_proxy
        if proxy is not None:
            proxy_url = urlparse.urlparse(proxy)
            if proxy_url.hostname is not None:
                proxy = (proxy_url.hostname, proxy_url.port)
            else:
                proxy = None
        return proxy


    def _test_mirror(self, mirror, dist):
        """Test if mirror is reachable
        """
        def _test_proxy(proxy_url):
            """
            wrapper for test_tcp call with port validation
            """
            hostname = proxy_url.hostname
            try:
                port = proxy_url.port
                assert port is not None
            except (ValueError, AssertionError):
                raise AptProxyError(_('Invalid port in proxy address: {}').format(proxy_url.netloc))
            return test_tcp(hostname, port)

        env_proxy = os.environ.get('http_proxy')
        if env_proxy:
            log.debug('using http_proxy={0} from environment'.format(env_proxy))
            proxy_url = urlparse.urlparse(env_proxy)
            if not _test_proxy(proxy_url):
                raise AptProxyError(_('cannot connect to environment proxy {0}').format(env_proxy))
        else:
            host = urlparse.urlparse(mirror).netloc
            host_proxy = apt_pkg.config.get("{0}::{1}".format(_APT_PROXY_OPT, host))
            if host_proxy and host_proxy != 'DIRECT':
                proxy_url = urlparse.urlparse(host_proxy)
                if not _test_proxy(proxy_url):
                    raise AptProxyError(_('cannot connect to proxy {0}').format(host_proxy))
                os.environ.update({'http_proxy': host_proxy})
            else:
                default_proxy = apt_pkg.config.get(_APT_PROXY_OPT)
                if default_proxy:
                    proxy_url = urlparse.urlparse(default_proxy)
                    if proxy_url.port == self.get_apt_cacher_port():
                        if not _test_proxy(proxy_url):
                            raise AptProxyError(_APT_CACHER_ERROR_MSG)
                        if self.is_container_mode() and \
                                self.client.get_creole('module_instancie') == 'oui' and \
                                not test_tcp(self.client.get_creole('adresse_ip_br0'), proxy_url.port):
                            raise AptCacherError(_APT_CACHER_ERROR_MSG_CONTAINER)
                    else:
                        if not _test_proxy(proxy_url):
                            raise AptProxyError(_('cannot connect to default proxy {0}').format(default_proxy))
                    os.environ.update({'http_proxy': default_proxy})

        release_url = '{0}/dists/{1}/Release'.format(mirror, dist)
        http = test_http(release_url, use_proxy=True)
        if not env_proxy:
            # reset environment
            os.environ.pop('http_proxy', None)
        if not http:
            msg = _('Unable to get repository Release: {0}')
            raise RepositoryError(msg.format(release_url))


    def _prepare_cache(self):
        self._load_apt_cache()

    def get_upgradable_list(self, silent=True):
        """Search all packages who needs to be updated.

        :return: upgradable packages
        :rtype: `list` of `apt.Package`

        """
        self._load_apt_cache()
        packages = []
        with self.cache.actiongroup():
            self.cache.upgrade(dist_upgrade=True)

            for package in self.cache.get_changes():
                if package.marked_delete:
                    # no candidate version for packages to remove (#10968)
                    packages.append([package.name, package.is_installed, None])
                else:
                    packages.append([package.name, package.is_installed, package.candidate.version])
            # Do not perform action
            self.cache.clear()

        return packages


    def _apply_to_package(self, action, packages, silent=False, simulate=False):
        """Perform an action on a package or package list.

        :param action: name of the action to perform on the package
        :type action: `str`
        :param packages: package name
        :type  packages: `str` or `list` of `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        check = {'install': True, 'remove': False}
        apply_to = []
        if not isinstance(packages, list):
            packages = [packages]

        self._load_apt_cache()
        with self.cache.actiongroup():
            for name in packages:
                pkg = self.get_package(name)
                # Mark the package for the action
                if pkg.is_installed != check[action] or \
                   (action == 'install' and not self._only_install):
                    apply_to.append(name)

            if not apply_to:
                log.info(_('No package to {0}.').format(action))
                return True

        args = [action]
        if simulate:
            args.insert(0, '-s')

        self._prepare_cache()
        code = _run_apt_get(args + apply_to, silent=silent)
        return code


    def install(self, packages, silent=False, simulate=False):
        """Install a package or a list of packages

        :param packages: package name
        :type  packages: `str` or `list` of `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        return self._apply_to_package('install', packages,
                                      silent=silent,
                                      simulate=simulate)


    def remove(self, packages, silent=False, simulate=False):
        """Remove a package

        :param packages: package name
        :type  packages: `str` or `list` of `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        return self._apply_to_package('remove', packages,
                                      silent=silent,
                                      simulate=simulate)


    def update(self, silent=False):
        """ update the package lists (same as apt-get update)

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        partial_dir = os.path.join(_LISTS_DIR, 'partial')
        if os.path.isdir(partial_dir):
            for filename in os.listdir(partial_dir):
                fullname = os.path.join(partial_dir, filename)
                log.debug('deleting partial file {0}'.format(fullname))
                os.unlink(fullname)

        self._prepare_cache()
        ret_code = _run_apt_get(['update'], silent=silent)
        if self.cache is not None:
            self.cache.open()
        return ret_code


    def check(self, silent=False):
        """ check dpkg errors

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        ret = system_out(['apt-get', 'install', '-f'])
        if ret[0] != 0:
            raise AptError (_(u"apt-get install -f returned an error\n{0}")\
                    .format(''.join(ret[1:])))
        return 0


    def upgrade(self, dist_upgrade=False, download_only=False, silent=False, simulate=False):
        """ upgrade all packages with new available versions (same as apt-get upgrade)

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        if dist_upgrade:
            action = ['dist-upgrade']
        else:
            action = ['upgrade']

        if download_only:
            action.insert(0, '-d')

        if simulate:
            action.insert(0, '-s')

        self._prepare_cache()
        code = _run_apt_get(action, silent=silent)

        if code == 0 and not download_only and not simulate:
            self.clean()

        return code


    def dist_upgrade(self, silent=False, simulate=False):
        """make dist-upgrade (same as apt-get dist-upgrade)

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        self.upgrade(dist_upgrade=True, silent=silent, simulate=simulate)


    def fetch_archives(self, packages, silent=False):
        """only download packages

        """
        self._prepare_cache()
        code = _run_apt_get(['-d', 'install'] + packages, silent=silent)
        return code


    def get_depends(self, pkgname):
        """Search dependencies for a package

        :param pkgname: package name
        :type  pkgname: `str`
        :return: dependencies for a package
        :rtype: `list` of `apt.Package`
        """
        depends = []
        self._load_apt_cache()
        pkg = self.get_package(pkgname)
        candidates = pkg.candidate.dependencies

        for dep in candidates:
            for elm in dep.or_dependencies:
                depends.append(elm.name)

        return depends


    def download(self, pkgname, dest='/tmp', silent=False):
        """Download a package binary

        The default destination is ``/tmp``.

        :param pkgname: package name
        :type  pkgname: `str`
        :param dest: destination directory
        :type dest: `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        old_dir = os.getcwd()
        os.chdir(dest)
        self._prepare_cache()
        ret = _run_apt_get(['download', pkgname], silent=silent)
        os.chdir(old_dir)
        return ret


    def clean(self):
        """Clean old deb file in cache dir

        see cmdline/apt-get.cc DoClean
        and apt-pkg/acquire.cc pkgAcquire::Clean
        """
        self._prepare_cache()
        code = _run_apt_get(['clean'])
        return code


class EolePkgYum(object):
    """yum wrapper class.
    This class mangage the interaction with "yum" based systems, like RHEL, CentOS or Fedora
    This is a 'private' class it is made to be used by :class:`EolePkg` with the 'yum'
    parameter

    - prototype: ``EolePkgYum(root)`` with ``root`` the root directory for
    package installation

    When :method:`__init__` is called you initialize a new interface for package
    management.

    """
    def __init__(self, root=None):
        raise NotImplementedError(_('  Yum Support is not Implemented yet'))


class EolePkg(object):
    """Main Eole package manipulation class
    Use this class to manipulate packages by passing the type to ``EolePkg()``

    This class manage automaticaly  the packages with the good backend.
    """
    def __init__(self, mode, root=None, test=False, container_mode=None,
                 only_install=True, ignore=False, is_maj_auto=False):
        """ Initialize the package manager interface
        :param mode: The backend type (apt, yum, urpmi)
        :type mode: string
        :param root: The system directory to be the root for package installation
        :type root: directory `string`
        :param test: used for dry run mode
        :type test: boolean
        :param only_install: install action does not update existing packages
        :type only_install: boolean
        """
        if mode == 'apt':
            self.pkgmgr = EolePkgApt(root=root,
                                     test=test,
                                     container_mode=container_mode,
                                     only_install=only_install, ignore=ignore)
        else:
            raise ImportError(_('No package manager module found'))

        self.ignore = ignore
        self._return_json = False

        self.set_test_run(_TESTRUN)
        self.set_test_run(test)


    def set_test_run(self, value):
        """Set a value to :attr:`self._testrun`
        Used by the unit test for dry run, don't use this
        """
        self._testrun = value
        self.pkgmgr.set_test_run(value)


    def set_option(self, option, value):
        """Set package manager option

        """
        self.pkgmgr.set_option(option, value)


    def set_json_output(self, root=None, container=None):
        """Use to get output of containers as json

        """
        # Return json only if asked on command line
        if root is not None:
            self._return_json = root
        if container is not None:
            self.pkgmgr._return_json = container


    def _run(self, action, func, container=None, **kwargs):
        """Run package manager actions for all containers

        Print a JSON dump of return value if asked.

        :param action: action to perform
        :type action: `str`
        :param func: package manager method to run
        :type func: `str`
        :param container: container name
        :type container: `str`
        :return: return values of commands by containers
        :rtype: `dict`

        """
        ret = {}

        if not self.pkgmgr.is_container_mode() or container in [None, 'all', 'root', 'current']:
            # Run for the current container
            if container != 'current' and not kwargs.get('silent', False):
                msg = _('Running {0} for root').format(action)
                log.info(msg)
            ret.update(self.pkgmgr._run(action=action,
                                        func=func,
                                        container=container,
                                        **kwargs))

        if container not in [None, 'root', 'current']:
            # Try for containers
            ret.update(self.pkgmgr._run_containers(action=action,
                                                   func=func,
                                                   container=container,
                                                   **kwargs))

        if self._return_json:
            print (json.dumps(ret))
        else:
            return ret


    def get_package(self, pkgname):
        """ get package search the package in package database.
        :param pkgname: package name
        :type: pkgname: `str`
        :return: a package from the package database.
        :rtype: `apt.Package` or `yum.Package`

        """
        return self.pkgmgr.get_package(pkgname)

    def get_rev_depends(self, pkgname):
        """ get package reverse depends list
        :param pkgname:  package name
        :type: pkgname: `str`
        :return: a package name list.
        :rtype: `list`
        """
        return self.pkgmgr.get_rev_depends(pkgname)


    def install(self, packages, container=None, silent=False, simulate=False):
        """Install a package

        :param packages: package names
        :type  packages: `str` or `list` of `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        action = 'install'
        func = 'install'

        if not isinstance(packages, list):
            packages = [packages]

        return self._run(action=action,
                         func=func,
                         packages=packages,
                         container=container,
                         silent=silent,
                         simulate=simulate)


    def remove(self, packages, container=None, silent=False, simulate=False):
        """Remove a package

        :param packages: package names
        :type  packages: `str` or `list` of `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        action = 'remove'
        func = 'remove'

        if not isinstance(packages, list):
            packages = [packages]

        return self._run(action=action,
                         func=func,
                         packages=packages,
                         container=container,
                         silent=silent,
                         simulate=simulate)


    def update(self, container=None, silent=False):
        """Update the package lists

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        action = 'update'
        func = 'update'

        if container is None:
            container = 'all'
        return self._run(action=action,
                         func=func,
                         container=container,
                         silent=silent)


    def check(self, container=None, silent=True):
        """Check dpkg errors

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        action = 'check'
        func = 'check'

        if container is None:
            container = 'all'
        return self._run(action=action,
                         func=func,
                         container=container,
                         silent=silent)

    def upgrade(self, container=None, dist=False, download_only=False,
                silent=False, simulate=False):
        """Upgrade all packages with new available versions

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        func = 'upgrade'
        action = 'upgrade'
        if dist:
            action = 'dist-upgrade'
        if download_only:
            action = 'download-upgrade'

        if container is None:
            container = 'all'

        return self._run(action=action,
                         func=func,
                         container=container,
                         dist_upgrade=dist,
                         download_only=download_only,
                         silent=silent, simulate=simulate)


    def download_upgrade(self, container=None, silent=False):
        """Upgrade the distribution

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        if container is None:
            container = 'all'

        return self.upgrade(container=container,
                            dist=True,
                            download_only=True,
                            silent=silent)

    def dist_upgrade(self, container=None, silent=False, simulate=False):
        """Upgrade the distribution

        :param silent: do not display progress informations
        :type silent: `bool`

        """
        if container is None:
            container = 'all'

        return self.upgrade(container=container,
                            dist=True,
                            silent=silent, simulate=simulate)

    def fetch_archives(self, packages, container=None, silent=False):
        """Download packages in apt cache
        """

        func = 'fetch_archives'
        action = 'fetch_archives'

        if container is None:
            container = 'all'

        if not isinstance(packages, list):
            packages = [packages]

        return self._run(action=action,
                         func=func,
                         packages=packages,
                         container=container,
                         silent=silent)


    def get_depends(self, pkgname=None):
        """Search dependencies for a package

        :param pkgname: package name
        :type  pkgname: `str`
        :return: dependencies for a package
        :rtype: `list` of `apt.Package` or `yum.Package`
        """
        return self.pkgmgr.get_depends(pkgname)


    def download(self, pkgname, dest='/tmp', silent=False):
        """Download a package binary

        The default destination is ``/tmp``.

        :param pkgname: package name
        :type  pkgname: `str`
        :param dest: destination directory
        :type dest: `str`
        :param silent: do not display progress informations
        :type silent: `bool`

        """
        return self.pkgmgr.download(pkgname, dest, silent=silent)


    def get_upgradable_list(self, container=None, silent=False):
        """ get_upgradable_list finds all packages who needs to be updated.
        """
        # Make sure to call the good apt-eole action in containers
        action = 'list-upgrade'
        func = 'get_upgradable_list'
        self.set_json_output(container=True)

        if container is None:
            container = 'all'

        ret = self._run(action=action,
                        func=func,
                        container=container,
                        silent=silent)

        self.set_json_output(container=False)
        return ret


    def list_upgrade(self, container=None, upgrades=None, silent=False):
        """List upgradable packages

        :param container: container name
        :type container: `str`
        :param upgrades: packages to upgrade/install, per container
        :type upgrades: `dict`
        :param silent: make APT calls silent
        :type silent: `bool`

        """
        msg = []
        if upgrades is None:
            upgrades = self.get_upgradable_list(container=container,
                                                silent=silent)

        if not isinstance(upgrades, dict):
            return

        combined = {'new': {}, 'upgrade': {}, 'delete': {}}
        for container, packages in upgrades.items():
            if not packages:
                continue
            for name, isInstalled, candidateVersion in packages:
                pkg_key = (name, candidateVersion)
                if isInstalled:
                    if candidateVersion is None:
                        if pkg_key not in combined['delete']:
                            combined['delete'][pkg_key] = [container]
                        else:
                            combined['delete'][pkg_key].append(container)
                    else:
                        if pkg_key not in combined['upgrade']:
                            combined['upgrade'][pkg_key] = [container]
                        else:
                            combined['upgrade'][pkg_key].append(container)
                else:
                    if pkg_key not in combined['new']:
                        combined['new'][pkg_key] = [container]
                    else:
                        combined['new'][pkg_key].append(container)

        if not combined['new'] and not combined['upgrade']:
            msg.append(_('No upgradable packages'))
        else:
            if combined['new']:
                msg.append(_('New packages:'))
                for pkg, containers in sorted(combined['new'].items()):
                    sorted_containers = ', '.join(_sort_containers(containers))
                    msg.append('    {0} ({1}) ({2})'.format(pkg[0], pkg[1], sorted_containers))
            if combined['upgrade']:
                msg.append(_('Upgradable packages:'))
                for pkg, containers in sorted(combined['upgrade'].items()):
                    sorted_containers = ', '.join(_sort_containers(containers))
                    msg.append('    {0} ({1}) ({2})'.format(pkg[0], pkg[1], sorted_containers))
            if combined['delete']:
                msg.append(_('Deletable packages:'))
                for pkg, containers in sorted(combined['delete'].items()):
                    sorted_containers = ', '.join(_sort_containers(containers))
                    msg.append('    {0} ({1})'.format(pkg[0], sorted_containers))

        return msg

    def clean(self):
        """Clean package in cache"""
        self.pkgmgr.clean()

# FIXME: Remove ignore argument since it's extracted from pkgmgr
def _configure_sources_mirror(pkgmgr, ubuntu=None, eole=None, envole=None,
                              level='stable', ignore=False,
                              release=None, envole_release=None, force_no_envole=False,
                              eole_level=None, envole_level=None, version=None):
    """Valid and generate sources.list for master and containers

    :param ubuntu: Ubuntu mirror to use
    :type pkgname: `str`
    :param eole: EOLE mirror to use
    :type pkgname: `str`
    :param envole: Envole mirror to use
    :type pkgname: `str`
    :param level: update's level
    :type level: `str`
    :param release: overwrite EOLE release
    :type release: `str`
    :param envole_release: overwrite Envole release
    :type envole_release: `str`
    :param eole_level: update's level for EOLE repository
    :type eole_level: `str`
    :param envole_level: update's level for Envole repository
    :type envole_level: `str`

    """

    def _test_mirror(vendor, mirrors):
        """Search first available mirror
        """
        dist = _MIRROR_DIST[vendor]
        checked_mirrors = []
        for mirror in mirrors:
            try:
                mirror_url = 'http://{0}/{1}'.format(mirror, vendor.lower())

                pkgmgr._test_mirror(mirror_url, dist)
                log.info(_('Configuring {0} with source {1}').format(vendor, mirror))
                checked_mirrors.append(mirror_url)
            except RepositoryError as err:
                msg = _('Unable to configure {0} mirror with {1}: {2}')
                msg = _('Not configuring {0} mirror with {1} which seems inaccessible: {2}')
                log.warning(msg.format(vendor, mirror, normalize(str(err))))
                continue
        if not checked_mirrors:
            msg = _('Unable to configure APT sources for {0}')
            raise RepositoryError(msg.format(vendor))
        return checked_mirrors

    def _do_template_deb822(ctx, engine, name):
        """Generate additional sources list file for the given context
        """
        source_sources_path = os.path.join(_SOURCES_LISTS_DIR, f"{name}.sources")
        file_info = {'name': source_sources_path,
                     'source': os.path.join(templatedir, 'source.sources'),
                     'full_name': os.path.join('/', ctx['path'],
                                               source_sources_path[1:]),
                     'activate' : True,
                     'del_comment': '',
                     'mkdir' : False,
                     'rm' : False}
        log.debug('Process template for {}'.format(file_info['full_name']))
        engine.prepare_template(file_info)
        engine.process(file_info, ctx)


    def _do_template_aptproxy(ctx):
        """Generate apt proxy configuration file for the given context
        """
        file_info = {'name': _APT_PROXY_FILE,
                     'source': os.path.join(templatedir, '02eoleproxy'),
                     'full_name': os.path.join('/', ctx['path'],
                                               _APT_PROXY_FILE[1:]),
                     'activate' : True,
                     'del_comment': '',
                     'mkdir' : False,
                     'rm' : False}
        log.debug('Process template for {}'.format(file_info['full_name']))
        engine.prepare_template(file_info)
        engine.process(file_info, ctx)

    def _do_template_eolehub(ctx, engine):
        """Generate container hub specific configuration
        """
        if not os.path.isfile(os.path.join(distrib_dir, 'eolehub.common.env')):
            return True

        file_info = {'name': _EOLEHUB_COMMON_ENV,
                     'source': os.path.join(templatedir, 'eolehub.common.env'),
                     'full_name': os.path.join('/', ctx['path'],
                                               _EOLEHUB_COMMON_ENV[1:]),
                     'activate': True,
                     'del_comment': '',
                     'mkdir': False,
                     'rm': False}
        log.debug('Process template for {}'.format(file_info['full_name']))
        engine.prepare_template(file_info)
        engine.process(file_info, ctx)

    # initialize template engine and context
    creoled_available = True

    if version is not None:
        eole_version = version
    else:
        eole_version = EOLE_VERSION

    if release is not None:
        eole_release = release
    else:
        eole_release = EOLE_RELEASE

    if eole_level is None:
        eole_level = level

    if envole_level is None:
        envole_level = level

    try:
        engine = CreoleTemplateEngine()
        groups = pkgmgr.get_groups()
        rootctx = groups['group_infos']['root']
    except CREOLE_ERRORS as err:
        if not pkgmgr.ignore:
            raise err
        creoled_available = False
        minimal_variables = {'ubuntu_version': UBUNTU_VERSION,
                             'eole_release': eole_release,
                             'eole_version': eole_version}
        engine = CreoleTemplateEngine(force_values=minimal_variables)
        rootctx = {'name': 'root',
                   'path': '/'}

    if creoled_available:
        _do_template_aptproxy(rootctx)
        # reload configuration (FIXME)
        apt_pkg.config.clear(_APT_PROXY_OPT)
        apt_pkg.read_config_dir(apt_pkg.config, _APT_CONFIG_DIR)

    # if ubuntu mirror is not forced
    if ubuntu is None:
        try:
            ubuntu = pkgmgr.client.get_creole('ubuntu_update_mirrors')
        except CREOLE_ERRORS as err:
            if pkgmgr.ignore:
                log.warning(_("Unable to find Creole variable for {0}, load default values: {1}").format('Ubuntu', _UBUNTU_MIRRORS))
                ubuntu = _UBUNTU_MIRRORS
            else:
                raise err
    else:
        ubuntu = [ubuntu]

    # test ubuntu mirror
    mirror_ubuntu = _test_mirror('Ubuntu', ubuntu)
    ubuntu_deb822_params = {
            '_deb822_keyname': '/usr/share/keyrings/ubuntu-archive-keyring.gpg',
            '_deb822_types': ['deb'],
            '_deb822_uris': mirror_ubuntu,
            '_deb822_suites': [f'{UBUNTU_VERSION}{suite}'
                               for suite in ['',
                                             '-security',
                                             '-updates',
                                             ]
                              ],
            '_deb822_components': ['main', 'universe', 'multiverse', 'restricted'],
            '_deb822_architectures': ['amd64'],
            }

    with tmp_engine_context(engine, ubuntu_deb822_params) as alt_engine:
        _do_template_deb822(rootctx, alt_engine, 'ubuntu')
    # if eole mirror is not forced
    if eole is None:
        try:
            eole = pkgmgr.client.get_creole(u'serveur_maj')
        except CREOLE_ERRORS as err:
            if pkgmgr.ignore:
                log.warning(_("Unable to find Creole variable for {0}, load default values: {1}").format('EOLE', _EOLE_MIRRORS))
                eole = _EOLE_MIRRORS
            else:
                raise err
    else:
        eole = [eole]
    # test eole mirror
    mirror_eole = _test_mirror('EOLE', eole)
    eole_deb822_params = {
            '_deb822_keyname': '/usr/share/keyrings/eole-archive-keyring.gpg',
            '_deb822_types': ['deb'],
            '_deb822_uris': mirror_eole,
            '_deb822_suites': {'stable': [f'eole-{eole_release}{suite}'
                                          for suite in ['',
                                                        '-security',
                                                        '-updates',
                                                        ]],
                               'proposed': [f'eole-{eole_release}{suite}'
                                            for suite in ['',
                                                          '-security',
                                                          '-updates',
                                                          '-proposed-updates'
                                                          ]],
                               'unstable': [f'eole-{eole_version}-unstable'],
                              }[eole_level],
            '_deb822_components': ['main', 'cloud'],
            '_deb822_architectures': ['amd64'],
            }

    # set container images tag (#34617, #35016)
    use_test_eole = any([m.startswith('http://test-eole.ac-dijon.fr') for m in mirror_eole])
    eolehub_tag = {'stable': 'stable',
                   'proposed': 'testing',
                   'unstable': 'dev'}['proposed'
                                      if use_test_eole and eole_level == 'stable'
                                      else 'unstable'
                                      if use_test_eole and eole_level == 'proposed'
                                      else eole_level]

    # Only if envole_version is defined
    envole_version = None
    if not force_no_envole:
        try:
            envole_version = pkgmgr.client.get_creole('envole_version', None)
        except CREOLE_ERRORS as err:
            pass

    if envole_version is not None:
        if envole_release is not None:
            envole_version = envole_release
        # if envole mirror is not forced
        if envole is None:
            try:
                envole = pkgmgr.client.get_creole('envole_update_mirrors')
            except CREOLE_ERRORS as err:
                if pkgmgr.ignore:
                    log.warning(_("Unable to find Creole variable for {0}, load default values: {1}").format('EOLE', _ENVOLE_MIRRORS))
                    envole = _ENVOLE_MIRRORS
                else:
                    raise err
        else:
            envole = [envole]
        # test envole mirror
        mirror_envole = _test_mirror('Envole', envole)
        envole_deb822_params = {
                '_deb822_keyname': f'/usr/share/keyrings/eole-archive-keyring.gpg',
                '_deb822_types': ['deb'],
                '_deb822_uris': mirror_envole,
                '_deb822_suites': {'stable': [f'envole-{envole_version}',],
                                   'proposed': [f'envole-{envole_version}{suite}'
                                                for suite in ['',
                                                              '-testing']],
                                   'unstable': [f'envole-{envole_version}-unstable'],
                                   }[envole_level],
                '_deb822_components': ['main'],
                '_deb822_architectures': ['amd64'],
                }
    else:
        mirror_envole = None


    try:
        with tmp_engine_context(engine, eole_deb822_params) as alt_engine:
            _do_template_deb822(rootctx, alt_engine, 'eole')
        if envole_version is not None:
            with tmp_engine_context(engine, envole_deb822_params) as alt_engine:
                _do_template_deb822(rootctx, alt_engine, 'envole')
        with tmp_engine_context(engine, {'_eolehub_tag': eolehub_tag}) as alt_engine:
            _do_template_eolehub(rootctx, alt_engine)
        if pkgmgr.is_container_mode():
            for group in groups['groups']:
                container_infos = groups['group_infos'][group]
                if group not in ['all', 'root'] and is_lxc_running(container_infos):
                    with tmp_engine_context(engine, eole_deb822_params) as alt_engine:
                        _do_template_deb822(container_infos, alt_engine, 'eole')
                    if mirror_envole:
                        with tmp_engine_context(engine, envole_deb822_params) as alt_engine:
                            _do_template_deb822(container_infos, alt_engine, 'envole')
                    if creoled_available:
                        _do_template_aptproxy(container_infos)
    except Exception as err:
        if pkgmgr.ignore:
            log.warning(_('cannot build sources.list use current one'))
        else:
            raise err
    client = CreoleClient()
    if creoled_available:
        additional_repository_name = client.get_creole('additional_repository_name')
        additional_repository_source_type = client.get_creole('additional_repository_source_type')
        additional_repository_source_uris = client.get_creole('additional_repository_source_uris')
        additional_repository_source_suites = client.get_creole('additional_repository_source_suites')
        additional_repository_source_components = client.get_creole('additional_repository_source_components')
        additional_repository_source_architectures = client.get_creole('additional_repository_source_architectures')
        additional_repository_key_type = client.get_creole('additional_repository_key_type')
        additional_repository_key_url = client.get_creole('additional_repository_key_url')
        additional_repository_key_signserver = client.get_creole('additional_repository_key_signserver')
        additional_repository_key_fingerprint = client.get_creole('additional_repository_key_fingerprint')
        for idx, name in enumerate(additional_repository_name):
            additional_repository_domain = urlparse.urlparse(additional_repository_source_uris[idx]).netloc
            filename = '/etc/apt/keyrings/{}.gpg'.format(name)
            log.info(_('Configuring {0} with source {1}').format(name, additional_repository_domain))
            if os.path.isfile(filename):
                code, stdin, stderr = system_out(["gpg","--import", "--dry-run", filename])
                if code != 0:
                    os.remove(filename)
            if not os.path.isfile(filename):
                key_type = additional_repository_key_type[idx]
                keyring_dir = tempfile.mkdtemp()
                keyring = os.path.join(keyring_dir, "pubring.gpg")
                if key_type == 'URL de la clé':
                    url = additional_repository_key_url[idx]
                    proxy = EolePkgApt._get_proxy(url)
                    if proxy is None:
                        env = None
                    else:
                        env = {'http_proxy': '{}:{}'.format(proxy[0], proxy[1]),
                               'https_proxy': '{}:{}'.format(proxy[0], proxy[1])}
                    logfile = '/tmp/{}.log'.format(name)
                    cmd = ['wget', url, '-o', logfile, '-O', keyring]
                    code = system_code(cmd, env=env)
                    if code != 0:
                        if os.path.isfile(keyring):
                            rmtree(keyring_dir)
                        raise AptProxyError(_('cannot retrieve the URL {}, more information in {}').format(url, logfile))
                else:
                    proxy = EolePkgApt._get_proxy()
                    keyserver = additional_repository_key_signserver[idx]
                    signing_key_fingerprint = additional_repository_key_fingerprint[idx]
                    secret_keyring = os.path.join(keyring_dir, "secring.gpg")
                    export_keyring = os.path.join(keyring_dir, "export-keyring.gpg")
                    cmd = ['gpg', '--no-default-keyring', '--no-options',
                           '--homedir', keyring_dir,
                           '--secret-keyring', secret_keyring,
                           '--keyring', export_keyring,
                           '--keyserver', keyserver]

                    if proxy is not None:
                        cmd.append('--keyserver-options')
                        cmd.append('http-proxy=http://{}:{}'.format(proxy[0], proxy[1]))

                    cmd.extend(['--recv', signing_key_fingerprint])

                    code, stdout, stderr = system_out(cmd)
                    if code != 0:
                        if os.path.isfile(keyring):
                            rmtree(keyring_dir)
                        raise AptProxyError(_('cannot retrieve the fingerprint {} from {}').format(signing_key_fingerprint, keyserver))
                    cmd = ['gpg', '--no-default-keyring', '--no-options',
                           '--homedir', keyring_dir,
                           '--keyring', export_keyring,
                           '--output', keyring,
                           '--armor',
                           '--export', signing_key_fingerprint]
                    code, stdout, stderr = system_out(cmd)
                    if code != 0:
                        if os.path.isfile(keyring):
                            rmtree(keyring_dir)
                        raise AptProxyError(_('cannot export the fingerprint {}').format(signing_key_fingerprint))
                if os.path.isfile(keyring):
                    cmd = ['gpg', '--dearmor', '-o', filename, keyring]
                    code, stdout, stderr = system_out(cmd)
                    if code != 0:
                        if os.path.isfile(filename):
                            os.remove(filename)
                        rmtree(keyring_dir)
                        raise AptProxyError(_('cannot import the key {}').format(name))
            try:
                if creoled_available:
                    deb822_variables = {
                                        '_deb822_keyname': filename,
                                        '_deb822_types': additional_repository_source_type[idx].split(),
                                        '_deb822_uris': additional_repository_source_uris[idx].split(),
                                        '_deb822_suites': additional_repository_source_suites[idx].split(),
                                        '_deb822_components': additional_repository_source_components[idx].split(),
                                        '_deb822_architectures': additional_repository_source_architectures[idx].split() if additional_repository_source_architectures[idx] else None,
                                        }
                    with tmp_engine_context(engine, deb822_variables) as alt_engine:
                        _do_template_deb822(rootctx, alt_engine, name)
                if pkgmgr.is_container_mode():
                    for group in groups['groups']:
                        container_infos = groups['group_infos'][group]
                        if group not in ['all', 'root'] and is_lxc_running(container_infos):
                            if creoled_available:
                                with tmp_engine_context(engine, deb822_variables) as alt_engine:
                                    _do_template_deb822(container_infos, alt_engine, name)
            except Exception as err:
                if pkgmgr.ignore:
                    log.warning(_(f'cannot build {name}.sources use current one'))
                else:
                    print(err)
                    raise err


    # delete old cache (#10186)
    pkgmgr.cache = None
