#!/usr/bin/python3 -u
# -*- mode: python; coding: utf-8 -*-
#
##########################################################################
# dl-iso - Download EOLE dictionaries
# Copyright © 2018 Pôle de compétences EOLE <eole@ac-dijon.fr>
# Author: Daniel Dehennin <daniel.dehennin@ac-dijon.fr>
#
# License CeCILL:
#  * in french: http://www.cecill.info/licences/Licence_CeCILL_V2-fr.html
#  * in english http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
##########################################################################

"""Download EOLE XML schemas

SYNOPSYS
========

    dl-schemas --release 2.7.0 --devel --verbose
    dl-schemas --release 2.6.0,2.6.1,2.6.2
    dl-schemas --release all
    dl-schemas --release 2.5.2 --out-dir /tmp/2.5.2/

"""

import os
import argparse

import requests

import shutil
import re

from debian import deb822
from debian import debfile
from debian.debian_support import AptPkgVersion

import logging

logger = None

EOLE_ENVOLE_MAPPING = {'2.4.0': 'envole-4',
                       '2.4.1': 'envole-4',
                       '2.4.2': 'envole-4',
                       '2.5.0': 'envole-4',
                       '2.5.1': 'envole-4',
                       '2.5.2': 'envole-5',
                       '2.6.0': 'envole-5',
                       '2.6.1': 'envole-6',
                       '2.6.2': 'envole-6',
                       '2.7.0': 'envole-6',
                       '2.7.1': 'envole-7',
                       '2.7.2': 'envole-7',
                       '2.8.0': 'envole-8',
                       '2.8.1': 'envole-8',
                       '2.9.0': 'envole-9',
}

EOLE_SUFFIXES = ['', '-updates', '-security']

EXCLUDED_PACKAGES = ['creole', # Provide only a sample dictionary
                     'eole-zephir', # Zephir can not manage another Zephir
                     ## MTES packages
                     'eole-antivir2',
                     'eole-asgard',
                     'ecdl-outils',
                     'eole-ecdl',
                     'eole-ecdlannuaire',
                     'eole-esbl',
                     'eole-esbl-glpi',
                     'eole-esbl-grr',
                     'eole-esbl-ocs',
                     'eole-geo-ide-base',
                     'eole-geo-ide-distribution',
                     'eole-ocsinventory-agent',
                     'eole-wapt',
                     'seth-ecologie',
                     'supervision-psin',
                     ## Envole packages
                     # 'eole-envole-migration',
]

EXCLUDED_SCHEMAS = []


def main():
    """Main execution function
    """
    options = parse_args()

    global logger
    logging.basicConfig(level=options.log_level.upper(),
                        style='$', # format style as string.Template
                        format=options.log_format)
    logger = logging.getLogger(name='dl-schema')

    if options.verbose:
        # Do not get urllib3 logs on verbose
        urllib3_logger = logging.getLogger(name='urllib3.connectionpool')
        urllib3_logger.propagate = False

    logger.debug(f'Start dl-schemas main procedure with options="{options}"')

    if not os.access(options.temp_dir, os.W_OK):
        logger.debug(f'Create temporary directory "{options.temp_dir}"')
        os.makedirs(options.temp_dir, 0o700)

    for release in options.release:
        logger.info(f'Release “{release}”: download EOLE XML schema files')
        loop_distributions(release, options)


def parse_args():
    """Simply parse the command line options

    """
    parser = argparse.ArgumentParser(description='Downoald EOLE dictionaries')
    parser.add_argument('-r', '--release',
                        nargs='*',
                        required=True,
                        type=str,
                        help='EOLE release')

    target_opt = parser.add_mutually_exclusive_group()
    target_opt.add_argument('-p', '--proposed',
                            action='store_true',
                            help='Use proposed-updates')

    target_opt.add_argument('--testing',
                            action='store_true',
                            help='Use testing distribution')

    target_opt.add_argument('--devel',
                            action='store_true',
                            help='Use devel distribution')

    parser.add_argument('-m', '--mirror',
                        type=str,
                        default='http://test-eole.ac-dijon.fr',
                        help='Debian package mirror')

    current_file = os.path.abspath(__file__)
    zephir_parc_dir = os.path.dirname(os.path.dirname(current_file))
    zephir_schema_dir = os.path.join(zephir_parc_dir, 'data', 'dictionnaires')
    parser.add_argument('--out-dir',
                        type=str,
                        default=zephir_schema_dir,
                        help='Download directory')

    parser.add_argument('--temp-dir',
                        type=str,
                        default='/tmp/dl-schemas',
                        help='Temporary download directory')

    parser.add_argument('--proxy', type=str, help="HTTP proxy")

    log_opts = parser.add_argument_group('logging')
    log_opts.add_argument('-l', '--log-level',
                          choices=['debug', 'info', 'warning', 'error', 'critical'],
                          default='info',
                          help='Log level')

    log_opts.add_argument('-v', '--verbose',
                          action='store_true',
                          help='Verbose mode, equivalent to -l info')

    log_opts.add_argument('-d', '--debug',
                          action='store_true',
                          help='Debug mode, equivalent to -l debug')

    options = parser.parse_args()

    ##
    ## Mangle options
    ##
    options.log_format = '${message}'
    if options.debug:
        options.log_level = 'debug'
        options.log_format = '${levelname}: ${message}'
    elif options.verbose:
        # Verbose is like debug without log level name prefix and urllib3 messages
        options.log_level = 'debug'

    if 'all' in options.release:
        options.release = sorted(EOLE_ENVOLE_MAPPING.keys())

    if not options.mirror.startswith('http://'):
        options.mirror = f'http://{options.mirror}'

    return options


def loop_distributions(release, options):
    """Loop on EOLE and Envole distributions

    :param str release: EOLE release
    :param Namespace options: options passed to the script
    """
    logger.debug(f'Release “{release}”: process all the distributions')

    eole_distributions = []
    eole_version = '.'.join(release.split('.')[0:2])

    if options.devel:
        eole_distributions = [f'eole-{eole_version}-unstable']
    elif options.testing:
        eole_distributions = [f'eole-{eole_version}-testing']
    else:
        suffixes = EOLE_SUFFIXES.copy()
        if options.proposed:
            suffixes.append('-proposed-updates')

        for suffix in suffixes:
            eole_distributions.append(f'eole-{release}{suffix}')

    if release in EOLE_ENVOLE_MAPPING and EOLE_ENVOLE_MAPPING[release]:
        envole_distribution = EOLE_ENVOLE_MAPPING[release]
        if options.devel:
            envole_distribution = f'{envole_distribution}-unstable'
        eole_distributions.append(envole_distribution)
    else:
        logger.warning(f'! No Envole release available for EOLE {release} !')

    download_eole_schemas(eole_distributions, release, options)
    copy_eole_schemas(release, options)


def download_eole_schemas(distributions, release, options):
    """Download the EOLE XML schemas for all the distributions

    :param list distributions: distributions where to look for EOLE XML schemas
    :param Namespace options: options passed to the script
    """
    logger.info(f'Release {release}: download the EOLE XML schemas for distributions “{", ".join(distributions)}”')
    logger.info(f'Using Debian package mirror: “{options.mirror}“')

    for distribution in distributions:
        logger.info(f'Release “{release}”: distribution “{distribution}”: download EOLE XML schemas')
        packages = get_packages(distribution, release, options)
        extract_eole_schemas(packages, distribution, release, options)


def copy_eole_schemas(release, options):
    """Copy EOLE XML schema files for a release to the output directory

    Copy the latest version of the EOLE XML schema for a release.

    :param str release: EOLE release
    :param Namespace options: options passed to the script
    """
    log_prefix = f'Release “{release}”:'
    logger.info(f'{log_prefix} copy latest EOLE XML schema files to output directory')

    source_schema_dir = os.path.join(options.temp_dir, 'schemas', release)

    if release == '2.4.0':
        release_schema_dir = os.path.join(options.out_dir, '2.4', 'eole')
    else:
        release_schema_dir = os.path.join(options.out_dir, release, 'eole')

    if os.access(release_schema_dir, os.W_OK):
        logger.debug(f'{log_prefix} delete existing release schema directory “{release_schema_dir}”')
        shutil.rmtree(release_schema_dir)

    logger.debug(f'{log_prefix} create release schema directory “{release_schema_dir}”')
    os.makedirs(release_schema_dir, 0o700)

    for package in os.listdir(source_schema_dir):
        package_log_prefix = f'{log_prefix} Package “{package}”:'
        logger.info(f'{package_log_prefix} copy EOLE XML schema')

        package_release_schema_dir = os.path.join(release_schema_dir, package)

        package_source_schema_dir = os.path.join(source_schema_dir, package)
        versions = os.listdir(package_source_schema_dir)
        logger.debug(f'{package_log_prefix} found versions “{versions}”')

        last_version = get_last_version(release, package, versions)
        last_version_dir = os.path.join(package_source_schema_dir, str(last_version))
        last_version_log_prefix = f'{package_log_prefix} Version “{last_version}”:'

        if os.access(package_release_schema_dir, os.W_OK):
            logger.debug(f'{last_version_log_prefix} delete existing package release directory “{package_release_schema_dir}”')
            shutil.rmtree(package_release_schema_dir)

        logger.debug(f'{last_version_log_prefix} create package release schema directory “{package_release_schema_dir}”')
        os.makedirs(package_release_schema_dir, 0o700)

        for schema in os.listdir(last_version_dir):
            source_schema_path = os.path.join(last_version_dir, schema)
            dest_schema_path = os.path.join(package_release_schema_dir, schema)
            logger.info(f'{last_version_log_prefix} copy EOLE XML schema file: “{source_schema_path}” => “{dest_schema_path}”')
            shutil.copyfile(source_schema_path, dest_schema_path)


def get_packages(distribution, release, options):
    """Retrieve the list of binary packages

    :param str distribution: distribution name
    :param str release: EOLE release
    :param Namespace options: options passed to the script
    """
    log_prefix = f'Release {release}: distribution {distribution}:'
    logger.debug(f'{log_prefix} get the binary packages')

    packages = []

    index_dir = os.path.join(options.temp_dir, 'indexes')
    if not os.access(index_dir, os.W_OK):
        logger.debug(f'{log_prefix} create temporary directory for indexes "{index_dir}"')
        os.makedirs(index_dir, 0o700)

    index_path = os.path.join(index_dir, f'{distribution}.Packages')
    vendor = distribution.split('-')[0]
    url = os.path.join(options.mirror,
                       vendor,
                       'dists',
                       distribution,
                       'main',
                       'binary-amd64',
                       'Packages')

    logger.debug(f'{log_prefix} download the package index: "{url}" => "{index_path}"')
    download_file(url, index_path)

    with open(index_path, 'r') as file_h:
        packages = [ pkg for pkg in deb822.Packages.iter_paragraphs(file_h) ]

    return packages


def extract_eole_schemas(packages, distribution, release, options):
    """Extract the EOLE schemas from downloaded binary packages

    :param list packages: packages of a distribution
    :param str distribution: name of the distribution
    :param str release: EOLE release
    :param Namespace options: options passed to the script
    """
    log_prefix = f'Release “{release}”: Distribution “{distribution}”:'
    vendor = distribution.split('-')[0]
    base_url = os.path.join(options.mirror, vendor)
    base_schema_dir = os.path.join(options.temp_dir, 'schemas', release)

    deb_dir = os.path.join(options.temp_dir, 'debs')
    if not os.access(deb_dir, os.W_OK):
        logger.debug(f'{log_prefix} create temporary directory for deb package file “{deb_dir}”')
        os.makedirs(deb_dir, 0o700)

    for package in packages:
        package_log_prefix = f'{log_prefix} Package “{package["Package"]}”: Version “{package["Version"]}”:'

        if 'Filename' not in package:
            logger.warning(f'{package_log_prefix} skip malformed package')
            continue
        elif package['Package'] in EXCLUDED_PACKAGES:
            logger.debug(f'{package_log_prefix} skip excluded package')
            continue

        logging.debug(f'{package_log_prefix} search EOLE XML schema files')

        url = os.path.join(base_url, package['Filename'])
        package_path = os.path.join(deb_dir, os.path.basename(url))
        download_file(url, package_path)

        deb = debfile.DebFile(package_path)

        package_schema_dir = os.path.join(base_schema_dir,
                                          package['Package'],
                                          package['Version'])

        if os.path.isdir(package_schema_dir):
            logger.debug(f'{log_prefix} delete existing directory {package_schema_dir}')
            shutil.rmtree(package_schema_dir)

        for schema in filter(is_eole_schema, list(deb.data)):

            schema_name = os.path.basename(schema)
            schema_path = os.path.join(package_schema_dir, schema_name)

            if schema_name in EXCLUDED_SCHEMAS:
                logger.debug(f'{package_log_prefix} skip excluded schema “{schema_name}”')
                continue

            logger.info(f'{package_log_prefix} extract “{schema_name}”')
            if not os.access(package_schema_dir, os.W_OK):
                logger.debug(f'{package_log_prefix} create schema directory “{package_schema_dir}”')
                os.makedirs(package_schema_dir, 0o700)

            with open(schema_path, 'wb') as schema_fh:
                logger.debug(f'{package_log_prefix} write EOLE XML schema “{schema}” to “{schema_path}”')
                schema_fh.write(deb.data.get_content(schema))

        if os.access(package_path, os.F_OK):
            os.unlink(package_path)


def is_eole_schema(filename):
    """Match on EOLE schema filename

    :param str filename: name of a file
    """
    return re.match(r'^.?/?usr/share/eole/creole/dicos/.*xml', filename)


def get_last_version(release, package, versions):
    """Return the latest version

    """
    log_prefix = f'Release “{release}”: Package “{package}”:'
    last_version = None

    for version in versions:
        deb_version = AptPkgVersion(version)
        if not last_version:
            logger.debug(f'{log_prefix} init last version with “{deb_version}”')
            last_version = deb_version
            continue
        elif deb_version > last_version:
            logger.debug(f'{log_prefix} replace version “{last_version}” with new version “{deb_version}”')
            last_version = deb_version
        else:
            logger.debug(f'{log_prefix} reject version “{deb_version}”: not newer than "{last_version}”')

    logger.debug(f'{log_prefix} found the latest version “{last_version}” for package “{package}”')

    return last_version


def download_file(url, filename):
    """Download a file with HTTP

    :param str url: URL of the file to download
    :param str filename: path where to store the content
    """
    response = requests.get(url, stream=True)

    if not response.ok:
        raise Exception(f'Can not download {url}: {response.reason}')

    with open(filename, 'wb') as file_h:
        shutil.copyfileobj(response.raw, file_h)


if __name__ == '__main__':
    main()
