#!/usr/bin/env python
"""
This module runs insights and serializes the results into a directory. It is
configurable with a yaml manifest that specifies what to load, what to run,
and what to serialize. If a manifest isn't provided, a default one is used that
runs all datasources in ``insights.specs.Specs`` and
``insights.specs.default.DefaultSpecs`` and saves all datasources in
``insights.specs.Specs``.
"""
from __future__ import print_function

import argparse
import logging
import os
import sys
import tempfile
from datetime import datetime

import yaml

from insights import apply_configs
from insights import apply_default_enabled
from insights import get_pool
from insights.core import blacklist
from insights.core import dr
from insights.core import filters
from insights.core.serde import Hydration
from insights.core.spec_cleaner import Cleaner
from insights.util import fs
from insights.util import utc
from insights.util.hostname import determine_hostname
from insights.util.subproc import call

SAFE_ENV = {
    "PATH": os.path.pathsep.join([
        "/bin",
        "/usr/bin",
        "/sbin",
        "/usr/sbin",
        "/usr/share/Modules/bin",
    ]),
    "LC_ALL": "C",
}

if "LANG" in os.environ:
    SAFE_ENV["LANG"] = os.environ["LANG"]

log = logging.getLogger(__name__)

default_manifest = """
---
# version is for the format of this file, not its contents.
version: 0

client:
  context:
    class: insights.core.context.HostContext
    args:
      timeout: 10 # timeout in seconds for commands. Doesn't apply to files.

  # commands and files to ignore
  blacklist:
    files: []
    commands: []
    patterns: []
    keywords: []

  # Can be a list of dictionaries with name/enabled fields or a list of strings
  # where the string is the name and enabled is assumed to be true. Matching is
  # by prefix, and later entries override previous ones. Persistence for a
  # component is disabled by default.
  persist:
    - name: insights.specs.Specs
      enabled: true

  run_strategy:
    name: serial
    args:
      max_workers: null

plugins:
  # disable everything by default
  # defaults to false if not specified.
  default_component_enabled: false

  # packages and modules to load
  packages:
    - insights.specs.default
    - insights.specs.datasources

  # configuration of loaded components. names are prefixes, so any component with
  # a fully qualified name that starts with a key will get the associated
  # configuration applied. Can specify timeout, which will apply to command
  # datasources. Can specify metadata, which must be a dictionary and will be
  # merged with the components' default metadata.
  configs:
    - name: insights.specs.Specs
      enabled: true
    - name: insights.specs.default.DefaultSpecs
      enabled: true
    - name: insights.specs.datasources
      enabled: true
    # needed for specs that aren't given names before used in DefaultSpecs
    - name: insights.core.spec_factory
      enabled: true

    # needed by multiple datasource specs/components
    - name: insights.parsers.hostname.Hostname
      enabled: true
    - name: insights.parsers.hostname.HostnameDefault
      enabled: true
    - name: insights.combiners.hostname.Hostname
      enabled: true
    - name: insights.parsers.uname.Uname
      enabled: true
    - name: insights.parsers.redhat_release.RedhatRelease
      enabled: true
    - name: insights.combiners.redhat_release.RedHatRelease
      enabled: true
    - name: insights.parsers.ps.PsAuxcww
      enabled: true
    - name: insights.parsers.ps.PsAuxww
      enabled: true
    - name: insights.combiners.ps.Ps
      enabled: true
    - name: insights.parsers.dmidecode.DMIDecode
      enabled: true
    - name: insights.parsers.installed_rpms.InstalledRpms
      enabled: true
    - name: insights.parsers.mount.ProcMounts
      enabled: true

    # needed for identifying RHEL major version
    - name: insights.components.rhel_version.IsRhel6
      enabled: true
    - name: insights.components.rhel_version.IsRhel7
      enabled: true
    - name: insights.components.rhel_version.IsRhel8
      enabled: true
    - name: insights.components.rhel_version.IsRhel9
      enabled: true

    # needed for cloud specs
    - name: insights.parsers.yum.YumRepoList
      enabled: true
    - name: insights.parsers.rhsm_conf.RHSMConf
      enabled: true
    - name: insights.combiners.cloud_provider.CloudProvider
      enabled: true
    - name: insights.components.cloud_provider.IsAWS
      enabled: true
    - name: insights.components.cloud_provider.IsAzure
      enabled: true
    - name: insights.components.cloud_provider.IsGCP
      enabled: true

    # needed for ceph specs
    - name: insights.components.ceph.IsCephMonitor
      enabled: true

    # needed for pcp specs
    - name: insights.parsers.systemd.unitfiles.UnitFiles
      enabled: true
    - name: insights.parsers.ros_config.RosConfig
      enabled: true

    # needed for 'teamdctl_config/state_dump' spec and nmcli_conn_show_uuids spec
    - name: insights.parsers.nmcli.NmcliConnShow
      enabled: true

    # needed for mssql_tls_cert_enddate
    - name: insights.parsers.mssql_conf.MsSQLConf
      enabled: true

    # needed for rsyslog_tls_cert_file
    - name: insights.parsers.rsyslog_conf.RsyslogConf
      enabled: true
    - name: insights.combiners.rsyslog_confs.RsyslogAllConf
      enabled: true

    # needed for sap specs
    - name: insights.parsers.saphostctrl.SAPHostCtrlInstances
      enabled: true
    - name: insights.combiners.sap.Sap
      enabled: true

    # needed for fw_security specs
    - name: insights.parsers.virt_what.VirtWhat
      enabled: true
    - name: insights.combiners.virt_what.VirtWhat
      enabled: true
    - name: insights.components.virtualization.IsBareMetal
      enabled: true

    # needed for 'modinfo_filtered_modules' spec
    - name: insights.parsers.lsmod.LsMod
      enabled: true

    # needed for satellite server specs
    - name: insights.combiners.satellite_version.SatelliteVersion
      enabled: true
    - name: insights.combiners.satellite_version.CapsuleVersion
      enabled: true
    - name: insights.components.satellite.IsSatellite
      enabled: true
    - name: insights.components.satellite.IsSatellite614AndLater
      enabled: true
    - name: insights.components.satellite.IsSatelliteLessThan614
      enabled: true
    - name: insights.components.satellite.IsSatellite611
      enabled: true
    - name: insights.components.satellite.IsCapsule
      enabled: true

    # needed for container specs
    - name: insights.parsers.podman_list.PodmanListContainers
      enabled: true
    - name: insights.parsers.docker_list.DockerListContainers
      enabled: true

    # needed for 'luks_data_sources' spec
    - name: insights.parsers.blkid.BlockIDInfo
      enabled: true
    - name: insights.components.cryptsetup.HasCryptsetupWithTokens
      enabled: true
    - name: insights.components.cryptsetup.HasCryptsetupWithoutTokens
      enabled: true

    # needed for 'iris' specs
    - name: insights.parsers.iris.IrisList
      enabled: true
    - name: insights.parsers.iris.IrisCpf
      enabled: true

    # needed for ausearch_insights_client
    - name: insights.components.rhel_version.IsGtOrRhel86
      enabled: true

    # needed for sealert spec
    - name: insights.parsers.selinux_config.SelinuxConfig
      enabled: true
    - name: insights.components.selinux.SELinuxEnabled
      enabled: true

    # needed for the 'fstab_mounted.dirs' to the 'ls_lan' spec
    - name: insights.parsers.fstab.FSTab
      enabled: true
""".strip()

EXCEPTIONS_TO_REPORT = set([
    OSError
])
"""Exception types that should be reported on after core collection."""


def load_manifest(data):
    """ Helper for loading a manifest yaml doc. """
    if isinstance(data, dict):
        return data
    if os.path.isfile(data):
        with open(data, 'r') as f:
            doc = yaml.safe_load(f)
    else:
        doc = yaml.safe_load(data)
    if not isinstance(doc, dict):
        raise Exception("Manifest didn't result in dict.")
    return doc


def load_packages(pkgs):
    for p in pkgs:
        dr.load_components(p, continue_on_error=False)


def apply_blacklist(cfg):

    def _skip_component(component):
        dr.set_enabled(component, enabled=False)
        blacklist.BLACKLISTED_SPECS.append(component.split('.')[-1])
        log.warning('WARNING: Skipping component: %s' % component)

    def _check_and_skip_component(name):
        def _isidentifier(name):
            if hasattr(str, 'isidentifier'):
                return name.isidentifier()
            else:
                return not any(it in name for it in ('/', ' ', '.', '-'))

        if not _isidentifier(name):
            # not symbolic name
            return False
        # like a symbolic name
        component = 'insights.specs.default.DefaultSpecs.{0}'.format(name)
        if dr.get_component_by_name(component):
            # is a sybmolic name
            _skip_component(component)
            return True
        # not symbolic name
        return False

    for fil in cfg.get("files", []):
        if not _check_and_skip_component(fil):
            blacklist.add_file(fil)

    for cmd in cfg.get("commands", []):
        if not _check_and_skip_component(cmd):
            blacklist.add_command(cmd)

    for component in cfg.get('components', []):
        if not dr.get_component_by_name(component):
            log.warning('WARNING: Unknown component in blacklist: %s' % component)
        else:
            _skip_component(component)

    if cfg.get('patterns'):
        log.warning("WARNING: Excluding patterns defined in blacklist configuration")

    if cfg.get('keywords'):
        log.warning("WARNING: Replacing keywords defined in blacklist configuration")


def create_context(ctx):
    """
    Loads and constructs the specified context with the specified arguments.
    If a '.' isn't in the class name, the 'insights.core.context' package is
    assumed.
    """
    ctx_cls_name = ctx.get("class", "insights.core.context.HostContext")
    if "." not in ctx_cls_name:
        ctx_cls_name = "insights.core.context." + ctx_cls_name
    ctx_cls = dr.get_component(ctx_cls_name)
    ctx_args = ctx.get("args", {})
    return ctx_cls(**ctx_args)


def get_to_persist(persisters):
    """
    Given a specification of what to persist, generates the corresponding set
    of components.
    """
    def specs():
        for p in persisters:
            if isinstance(p, dict):
                yield p["name"], p.get("enabled", True)
            else:
                yield p, True

    components = sorted(dr.DELEGATES, key=dr.get_name)
    names = dict((c, dr.get_name(c)) for c in components)

    results = set()
    for p, e in specs():
        for c in components:
            if names[c].startswith(p):
                if e:
                    results.add(c)
                elif c in results:
                    results.remove(c)
    return results


def create_archive(path, remove_path=True):
    """
    Creates a tar.gz of the path using the path basename + "tar.gz"
    The resulting file is in the parent directory of the original path, and
    the original path is removed.
    """
    root_path = os.path.dirname(path)
    relative_path = os.path.basename(path)
    archive_path = path + ".tar.gz"

    cmd = [["tar", "-C", root_path, "-czf", archive_path, relative_path]]
    call(cmd, env=SAFE_ENV)
    if remove_path:
        fs.remove(path)
    return archive_path


def generate_archive_name():
    """
    Creates the archive directory to store the component output.
    """
    hostname = determine_hostname()
    suffix = datetime.now(utc).strftime("%Y%m%d%H%M%S")
    return "insights-%s-%s" % (hostname, suffix)


def collect(client_config=None, rm_conf=None, tmp_path=None,
            archive_name=None, compress=False, manifest=None):
    """
    This is the collection entry point. It accepts a manifest, a temporary
    directory in which to store output, and a boolean for optional compression.

    Args:
        client_config (InsightsConfig): Configurations read from insights-client
            configuration, including "manifest".
        rm_conf (dict): Client-provided python dict containing keys
            "commands", "files", and "keywords", to be injected
            into the manifest blacklist.
        tmp_path (str): The temporary directory that is used to create a
            working directory for storing the final tar.gz if one is generated.
        archive_name (str): The directory that is used to generate the output
            as well as the final tar.gz.
        compress (boolean): True to create a tar.gz and remove the original
            workspace containing output. False to leave the workspace without
            creating a tar.gz
        manifest (str or dict): json document or dictionary containing the
            collection manifest. See default_manifest for an example.  This
            option works only for `insights-collect` where 'client_config'
            is not filled.

    Returns:
        (str, dict): The full path to the created tar.gz or workspace.
        And a dictionary with relevant exceptions captured by the broker during
        core collection, this dictionary has the following structure:
        ``{ exception_type: [ (exception_obj, component), (exception_obj, component) ]}``.
    """
    # Get the manifest per the following order:
    # 1. "client_config.manifest"
    # 2. "manifest" passed to the `insights.collect()`
    # 3. "default_manifest" defined in this module
    manifest = default_manifest if manifest is None else manifest
    if client_config and hasattr(client_config, 'manifest') and client_config.manifest:
        manifest = client_config.manifest
    manifest = load_manifest(manifest)

    client = manifest.get("client", {})
    plugins = manifest.get("plugins", {})

    load_packages(plugins.get("packages", []))
    apply_default_enabled(plugins)
    apply_configs(plugins)
    # process blacklist
    black_list = client.get("blacklist", {})
    black_list.update(rm_conf or {})
    apply_blacklist(black_list)

    # insights-client
    if client_config and client_config.cmd_timeout:
        try:
            client['context']['args']['timeout'] = client_config.cmd_timeout
        except LookupError:
            log.warning('Could not set timeout option.')

    try:
        filters.load()
    except IOError as e:
        # could not load filters file
        log.debug("No filters available: %s", str(e))
    except AttributeError as e:
        # problem parsing the filters
        log.debug("Could not parse filters: %s", str(e))

    output_path = os.path.join(tmp_path, archive_name)
    fs.ensure_path(output_path)
    fs.touch(os.path.join(output_path, "insights_archive.txt"))

    broker = dr.Broker()
    ctx = create_context(client.get("context", {}))
    cleaner = Cleaner(client_config, black_list) if client_config else None
    broker[ctx.__class__] = ctx
    broker['cleaner'] = cleaner
    broker['redact_config'] = black_list
    broker['client_config'] = client_config

    # run in "serial" mode by default
    run_strategy = client.get("run_strategy", {"name": "serial"})
    parallel = run_strategy.get("name") == "parallel"
    to_persist = get_to_persist(client.get("persist", set()))

    if client_config and client_config.obfuscate and parallel:
        log.warning("Parallel collection is not supported when 'obfuscate' is enabled")
        parallel = False

    pool_args = run_strategy.get("args", {})
    with get_pool(parallel, "insights-collector-pool", pool_args) as pool:
        h = Hydration(output_path, ctx, pool=pool)
        broker.add_observer(h.make_persister(to_persist))
        dr.run_all(broker=broker, pool=pool)

    collect_errors = _parse_broker_exceptions(broker, EXCEPTIONS_TO_REPORT)

    cleaner.generate_report(archive_name) if cleaner else None

    if compress:
        return create_archive(output_path), collect_errors
    return output_path, collect_errors


def _parse_broker_exceptions(broker, exceptions_to_report):
    """
    Parse the exceptions captured in the broker during core collection
    and keep only exception types configured in the ``exceptions_to_report``.

    Args:
        broker (Broker): Broker object used for core collection.
        exceptions_to_report (set): Exception types to retrieve from the broker.

    Returns:
        (dict): A dictionary with relevant exceptions captured by the broker.
    """
    errors = {}
    try:
        if broker.exceptions:
            for component, exceptions in broker.exceptions.items():
                for ex in exceptions:
                    ex_type = type(ex)
                    if ex_type in exceptions_to_report:
                        if ex_type in errors.keys():
                            errors[ex_type].append((ex, component))
                        else:
                            errors[ex_type] = [(ex, component)]
    except Exception as e:
        log.warning("Could not parse exceptions from the broker.: %s", str(e))
    return errors


def main():
    # Remove command line args so that they are not parsed by any called modules
    # The main fxn is only invoked as a cli, if calling from another cli then
    # use the collect function instead
    collect_args = [arg for arg in sys.argv[1:]] if len(sys.argv) > 1 else []
    sys.argv = [sys.argv[0], ] if sys.argv else sys.argv

    p = argparse.ArgumentParser()
    p.add_argument("-m", "--manifest", help="Manifest yaml.")
    p.add_argument("-o", "--out_path", help="Path to write output data.")
    p.add_argument("-q", "--quiet", help="Error output only.", action="store_true")
    p.add_argument("-v", "--verbose", help="Verbose output.", action="store_true")
    p.add_argument("-d", "--debug", help="Debug output.", action="store_true")
    p.add_argument("-c", "--compress", help="Compress", action="store_true")
    args = p.parse_args(args=collect_args)

    level = logging.WARNING
    if args.verbose:
        level = logging.INFO
    if args.debug:
        level = logging.DEBUG
    if args.quiet:
        level = logging.ERROR

    logging.basicConfig(level=level)

    out_path = args.out_path or tempfile.gettempdir()
    archive, errors = collect(manifest=args.manifest, tmp_path=out_path,
                              archive_name=generate_archive_name(),
                              compress=args.compress)
    print(archive)


if __name__ == "__main__":
    main()
