#!/usr/bin/env python3
# -*- coding: utf-8 -*- vim60:fdm=marker
#
# Copyright: 2016, Maximiliano Curia <maxy@debian.org>
#
# License: ISC
#  Permission to use, copy, modify, and/or distribute this software for any
#  purpose with or without fee is hereby granted, provided that the above
#  copyright notice and this permission notice appear in all copies.
#  .
#  THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
#  REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
#  AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
#  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
#  LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
#  OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
#  PERFORMANCE OF THIS SOFTWARE.

'''File Parsers'''

import codecs
import collections
import functools
import html
import io
import logging
import shutil
import string
import subprocess
import tokenize

try:
    import regex as re
except ImportError:
    import re
import re as reo

import xdg.Mime

# Local modules
from .matchers import find_licenses, clean_comments, parse_holders

# initialize xdg cache before using it
xdg.Mime.update_cache()

PRINTABLE_BYTES = bytes(string.printable, encoding='ascii')


def generic_input(reader):

    def detect_bom(buffer):
        # This is what chardet.detect does, but chardet fallsback to ascii. :/
        # So you either need to feed all the file to chardet and read it
        # twice (or keep it in memory)
        for encoding, boms in (
                ('UTF-8-SIG', (codecs.BOM_UTF8,)),
                ('UTF-32', (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)),
                ('X-ISO-10646-UCS-4-3412', (b'\xFE\xFF\x00\x00',)),
                ('X-ISO-10646-UCS-4-2143', (b'\x00\x00\xFF\xFE',)),
                ('UTF-16', (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)),
        ):
            if buffer.startswith(boms):
                return encoding
        return 'utf-8'

    buffer = reader.peek(4)
    default_encoding = detect_bom(buffer)
    fallback_encoding = 'latin1'

    for raw_line in reader:
        try:
            line = raw_line.decode(default_encoding)
        except UnicodeDecodeError:
            if has_control_chars(raw_line):
                # binary?
                chars = (chr(char) if char in PRINTABLE_BYTES else '\n'
                         for char in raw_line)
                line = ''.join(chars)
                # Drop extra new lines
                line = reo.sub(r'\n{2,}', '\n', line)
            else:
                line = raw_line.decode(fallback_encoding)
        # convert mac to unix
        line = reo.sub(r'\r([^\n])', r'\n\1', line)
        yield line


def _get_content(filename):
    'Get the whole file content'

    content = []

    with open(filename, 'rb') as f:
        for line in generic_input(f):
            content.append(line)

    return ''.join(content)


def _process_content(content):
    'Process file contents'

    holders = parse_holders(content)
    content = clean_comments(content)

    licenses = find_licenses(content)

    return holders, licenses


def generic_parser(filename):
    '''Generic read, treat all files as text

    Binary content is filtered ala 'strings(1)'.
    Bom is detected, files with no bom are treated as utf-8 files, but lines
    that fail to be decoded are decoded with latin1.
    '''

    content = _get_content(filename)
    return _process_content(content)


@functools.lru_cache()
def cmd_parser_factory(*cmd):

    def _parser(filename):

        fullcmd = cmd + (filename,)

        try:
            result = subprocess.run(
                fullcmd, check=True,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            # A bit overkill?
            content = ''.join(
                generic_input(io.BufferedReader(io.BytesIO(result.stdout)))
            )
            content += ''.join(
                generic_input(io.BufferedReader(io.BytesIO(result.stderr)))
            )
        except (subprocess.CalledProcessError, UnicodeDecodeError) as e:
            logging.info('failed to parse %s with %s, (%s)',
                         filename, cmd, e)
            logging.info('falling back')
            return generic_parser(filename)

        return _process_content(content)

    if shutil.which(cmd[0]) is None:
        logging.warn('command %s not found, using generic parser as fallback',
                     cmd[0])
        return generic_parser

    return _parser


def html_parser(filename):
    'Drop/convert html entities, and brs'

    # Keep in mind that most template systems will also match this type. We
    # could use beautifulsoup to obtain the text and the html comments, but we
    # would loose the templating system comments, and the comments of the js
    # parts.

    content = _get_content(filename)
    # Not sure if this should go after the unescape
    content = reo.sub(r'<\s*br\s*/?>', '\n', content)
    try:
        content = html.unescape(content)
    except Exception as e:
        logging.info('failed to unescape html entities from %s, (%s)',
                     filename, e)
        logging.info('using content as is')

    return _process_content(content)


POUND_LINE_RE = re.compile(r'^\s*#')


def pound_lines_parser(filename):
    '''Keep only the lines starting with #'''

    lines = []
    with open(filename, 'rb') as f:

        for line in generic_input(f):
            if POUND_LINE_RE.match(line):
                lines.append(line)
            else:
                lines.append('\n')

    content = ''.join(lines)

    return _process_content(content)


PO_COMMENTED_TRANSLATION_RE = re.compile(r'^\s*#.{,2} msg(?:str|id|ctxt)\b')


def po_file_parser(filename):
    '''Keep only the comments (ignore also #~ msgid lines)'''

    lines = []
    with open(filename, 'rb') as f:

        for line in generic_input(f):
            if not POUND_LINE_RE.match(line):
                lines.append('\n')
            elif PO_COMMENTED_TRANSLATION_RE.match(line):
                lines.append('\n')
            else:
                lines.append(line)

    content = ''.join(lines)

    return _process_content(content)


def python_parser(filename):
    '''Extract comments and doc strings from a python file'''

    lines = []
    newline = True
    with open(filename, 'rb') as f:
        for token in tokenize.tokenize(f.readline):
            if (token.type == tokenize.COMMENT) or \
               (newline and token.type == tokenize.STRING):
                lines.append(token.string)
            elif token.type == tokenize.NEWLINE:
                newline = True
            elif newline and \
               (token.type in {tokenize.INDENT, tokenize.NL, tokenize.ENCODING}):
                continue
            else:
                newline = False
    content = '\n'.join(lines)

    return _process_content(content)


KNOWN_PARSERS = {
    'application/gzip': cmd_parser_factory('zcat'),
    'application/x-bzip': cmd_parser_factory('bzcat'),
    'application/x-lzma': cmd_parser_factory('xzcat'),
    'application/x-xz': cmd_parser_factory('xzcat'),
    'image/svg+xml-compressed': cmd_parser_factory('zcat'),

    # 'application/bzip2': cmd_parser_factory('exiftool'),
    # 'application/json': cmd_parser_factory('exiftool'),
    # 'application/octet-stream': cmd_parser_factory('exiftool'),
    # 'application/postscript': cmd_parser_factory('exiftool'),
    # 'application/rdf+xml': cmd_parser_factory('exiftool'),
    # 'application/x-bittorrent': cmd_parser_factory('exiftool'),
    # 'application/x-gzip': cmd_parser_factory('exiftool'),
    # 'application/x-httpd-php': cmd_parser_factory('exiftool'),
    # 'application/x-rar-compressed': cmd_parser_factory('exiftool'),
    # 'application/x-tar': cmd_parser_factory('exiftool'),
    # 'application/xml': cmd_parser_factory('exiftool'),
    # 'application/zip': cmd_parser_factory('exiftool'),
    # 'image/svg+xml': cmd_parser_factory('exiftool'),
    # 'text/calendar': cmd_parser_factory('exiftool'),
    # 'text/html': cmd_parser_factory('exiftool'),
    # 'text/rtf': cmd_parser_factory('exiftool'),
    # 'text/vcard': cmd_parser_factory('exiftool'),
    'application/ResEdit': cmd_parser_factory('exiftool'),
    'application/dicom': cmd_parser_factory('exiftool'),
    'application/itunes': cmd_parser_factory('exiftool'),
    'application/msword': cmd_parser_factory('exiftool'),
    'application/mxf': cmd_parser_factory('exiftool'),
    'application/pdf': cmd_parser_factory('exiftool'),
    'application/sketch': cmd_parser_factory('exiftool'),
    'application/vnd.adobe.fla': cmd_parser_factory('exiftool'),
    'application/vnd.adobe.illustrator': cmd_parser_factory('exiftool'),
    'application/vnd.adobe.indesign-idml-package': cmd_parser_factory('exiftool'),
    'application/vnd.adobe.photoshop': cmd_parser_factory('exiftool'),
    'application/vnd.iccprofile': cmd_parser_factory('exiftool'),
    'application/vnd.ms-excel': cmd_parser_factory('exiftool'),
    'application/vnd.ms-excel.addin.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-excel.sheet.binary.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-excel.sheet.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-excel.template.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-officetheme': cmd_parser_factory('exiftool'),
    'application/vnd.ms-powerpoint': cmd_parser_factory('exiftool'),
    'application/vnd.ms-powerpoint.presentation.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-powerpoint.slideshow.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-powerpoint.template.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-word.document.macroEnabled': cmd_parser_factory('exiftool'),
    'application/vnd.ms-word.template.macroEnabledTemplate': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.chart': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.database': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.formula': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.graphics': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.image': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.presentation': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.spreadsheet': cmd_parser_factory('exiftool'),
    'application/vnd.oasis.opendocument.text': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.presentationml.presentation': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.presentationml.slideshow': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.presentationml.template': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.spreadsheetml.template': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': cmd_parser_factory('exiftool'),
    'application/vnd.openxmlformats-officedocument.wordprocessingml.template': cmd_parser_factory('exiftool'),
    'application/vnd.palm': cmd_parser_factory('exiftool'),
    'application/vnd.rn-realmedia': cmd_parser_factory('exiftool'),
    'application/vnd.rn-realmedia-vbr': cmd_parser_factory('exiftool'),
    'application/x-3ds': cmd_parser_factory('exiftool'),
    'application/x-captureone': cmd_parser_factory('exiftool'),
    'application/x-chm': cmd_parser_factory('exiftool'),
    'application/x-dfont': cmd_parser_factory('exiftool'),
    'application/x-font-otf': cmd_parser_factory('exiftool'),
    'application/x-font-ttf': cmd_parser_factory('exiftool'),
    'application/x-font-type1': cmd_parser_factory('exiftool'),
    'application/x-indesign': cmd_parser_factory('exiftool'),
    'application/x-indesign-interchange': cmd_parser_factory('exiftool'),
    'application/x-iso9660-image': cmd_parser_factory('exiftool'),
    'application/x-iwork-keynote-sffkey': cmd_parser_factory('exiftool'),
    'application/x-iwork-numbers-sffnumbers': cmd_parser_factory('exiftool'),
    'application/x-iwork-pages-sffpages': cmd_parser_factory('exiftool'),
    'application/x-magick-image': cmd_parser_factory('exiftool'),
    'application/x-matroska': cmd_parser_factory('exiftool'),
    'application/x-mie': cmd_parser_factory('exiftool'),
    'application/x-mobipocket-ebook': cmd_parser_factory('exiftool'),
    'application/x-shockwave-flash': cmd_parser_factory('exiftool'),
    'application/x-visio': cmd_parser_factory('exiftool'),
    'application/x-wmf': cmd_parser_factory('exiftool'),
    'audio/audible': cmd_parser_factory('exiftool'),
    'audio/flac': cmd_parser_factory('exiftool'),
    'audio/mpeg': cmd_parser_factory('exiftool'),
    'audio/ogg': cmd_parser_factory('exiftool'),
    'audio/x-aiff': cmd_parser_factory('exiftool'),
    'audio/x-ds2': cmd_parser_factory('exiftool'),
    'audio/x-dss': cmd_parser_factory('exiftool'),
    'audio/x-matroska': cmd_parser_factory('exiftool'),
    'audio/x-monkeys-audio': cmd_parser_factory('exiftool'),
    'audio/x-ms-wma': cmd_parser_factory('exiftool'),
    'audio/x-musepack': cmd_parser_factory('exiftool'),
    'audio/x-pn-realaudio': cmd_parser_factory('exiftool'),
    'audio/x-pn-realaudio-plugin': cmd_parser_factory('exiftool'),
    'font/ttf': cmd_parser_factory('exiftool'),
    'image/apng': cmd_parser_factory('exiftool'),
    'image/bmp': cmd_parser_factory('exiftool'),
    'image/bpg': cmd_parser_factory('exiftool'),
    'image/flif': cmd_parser_factory('exiftool'),
    'image/gif': cmd_parser_factory('exiftool'),
    'image/jng': cmd_parser_factory('exiftool'),
    'image/jp2': cmd_parser_factory('exiftool'),
    'image/jpeg': cmd_parser_factory('exiftool'),
    'image/jpm': cmd_parser_factory('exiftool'),
    'image/jpx': cmd_parser_factory('exiftool'),
    'image/pgf': cmd_parser_factory('exiftool'),
    'image/pict': cmd_parser_factory('exiftool'),
    'image/png': cmd_parser_factory('exiftool'),
    'image/tiff': cmd_parser_factory('exiftool'),
    'image/vnd.djvu': cmd_parser_factory('exiftool'),
    'image/vnd.fpx': cmd_parser_factory('exiftool'),
    'image/vnd.ms-photo': cmd_parser_factory('exiftool'),
    'image/vnd.radiance': cmd_parser_factory('exiftool'),
    'image/x-adobe-dng': cmd_parser_factory('exiftool'),
    'image/x-canon-cr2': cmd_parser_factory('exiftool'),
    'image/x-canon-cr3': cmd_parser_factory('exiftool'),
    'image/x-canon-crw': cmd_parser_factory('exiftool'),
    'image/x-dpx': cmd_parser_factory('exiftool'),
    'image/x-epson-erf': cmd_parser_factory('exiftool'),
    'image/x-exr': cmd_parser_factory('exiftool'),
    'image/x-exv': cmd_parser_factory('exiftool'),
    'image/x-fujifilm-raf': cmd_parser_factory('exiftool'),
    'image/x-gopro-gpr': cmd_parser_factory('exiftool'),
    'image/x-hasselblad-3fr': cmd_parser_factory('exiftool'),
    'image/x-hasselblad-fff': cmd_parser_factory('exiftool'),
    'image/x-j2c': cmd_parser_factory('exiftool'),
    'image/x-kodak-dcr': cmd_parser_factory('exiftool'),
    'image/x-kodak-k25': cmd_parser_factory('exiftool'),
    'image/x-kodak-kdc': cmd_parser_factory('exiftool'),
    'image/x-leica-rwl': cmd_parser_factory('exiftool'),
    'image/x-light-lri': cmd_parser_factory('exiftool'),
    'image/x-lytro-lfp': cmd_parser_factory('exiftool'),
    'image/x-mamiya-mef': cmd_parser_factory('exiftool'),
    'image/x-minolta-mrw': cmd_parser_factory('exiftool'),
    'image/x-nikon-nef': cmd_parser_factory('exiftool'),
    'image/x-nikon-nrw': cmd_parser_factory('exiftool'),
    'image/x-olympus-orf': cmd_parser_factory('exiftool'),
    'image/x-paintshoppro': cmd_parser_factory('exiftool'),
    'image/x-panasonic-rw2': cmd_parser_factory('exiftool'),
    'image/x-pentax-pef': cmd_parser_factory('exiftool'),
    'image/x-photo-cd': cmd_parser_factory('exiftool'),
    'image/x-portable-bitmap': cmd_parser_factory('exiftool'),
    'image/x-portable-graymap': cmd_parser_factory('exiftool'),
    'image/x-portable-pixmap': cmd_parser_factory('exiftool'),
    'image/x-quicktime': cmd_parser_factory('exiftool'),
    'image/x-raw': cmd_parser_factory('exiftool'),
    'image/x-rawzor': cmd_parser_factory('exiftool'),
    'image/x-samsung-srw': cmd_parser_factory('exiftool'),
    'image/x-sigma-x3f': cmd_parser_factory('exiftool'),
    'image/x-sony-arw': cmd_parser_factory('exiftool'),
    'image/x-sony-sr2': cmd_parser_factory('exiftool'),
    'image/x-sony-srf': cmd_parser_factory('exiftool'),
    'image/x-tiff-big': cmd_parser_factory('exiftool'),
    'image/x-xcf': cmd_parser_factory('exiftool'),
    'video/3gp': cmd_parser_factory('exiftool'),
    'video/3gpp': cmd_parser_factory('exiftool'),
    'video/3gpp2': cmd_parser_factory('exiftool'),
    'video/divx': cmd_parser_factory('exiftool'),
    'video/m2ts': cmd_parser_factory('exiftool'),
    'video/mng': cmd_parser_factory('exiftool'),
    'video/mp4': cmd_parser_factory('exiftool'),
    'video/mpeg': cmd_parser_factory('exiftool'),
    'video/ogg': cmd_parser_factory('exiftool'),
    'video/quicktime': cmd_parser_factory('exiftool'),
    'video/vnd.rn-realvideo': cmd_parser_factory('exiftool'),
    'video/webm': cmd_parser_factory('exiftool'),
    'video/x-canon-crm': cmd_parser_factory('exiftool'),
    'video/x-dv': cmd_parser_factory('exiftool'),
    'video/x-flv': cmd_parser_factory('exiftool'),
    'video/x-matroska': cmd_parser_factory('exiftool'),
    'video/x-ms-asf': cmd_parser_factory('exiftool'),
    'video/x-ms-dvr': cmd_parser_factory('exiftool'),
    'video/x-ms-wmv': cmd_parser_factory('exiftool'),
    'video/x-ms-wtv': cmd_parser_factory('exiftool'),
    'video/x-red-r3d': cmd_parser_factory('exiftool'),

    'text/x-gettext-translation': po_file_parser,
    'text/x-gettext-translation-template': po_file_parser,
    'text/x-po': po_file_parser,
    'text/x-pot': po_file_parser,

    'text/x-python': python_parser,

    'text/html': html_parser,
    'application/x-php': html_parser,
}


def has_control_chars(bytes_seq):
    for i in bytes_seq:
        # ascii control chars (C0), and delete (DEL)
        if i < 32 or i == 127:
            # except tab (HT), line feed (LF) and carriage return (CR)
            if i in (9, 10, 13):
                continue
            return True
    return False


def lookup_mimetypes(xdg_type):

    queue = collections.deque()
    queue.append(xdg_type)
    seen = set()
    while queue:
        xdg_mime = queue.popleft()
        mime = str(xdg_mime)

        if mime in seen:
            continue
        seen.add(mime)

        parser = KNOWN_PARSERS.get(mime)
        if parser:
            return parser

        queue.extend(xdg_mime.inherits_from())
        logging.debug("Original type: %s, queue: %s", xdg_type, queue)

    return generic_parser


def parse_file(fullname, options):
    '''Parses the received file with the matching parser.

    Returns:
        The list of copyrights and licenses obtained by the parser.
    '''
    if options.text:
        parser = generic_parser
    else:
        xdg_type = xdg.Mime.get_type(fullname)
        logging.debug('Type for %s is: %s', fullname, xdg_type)

        parser = lookup_mimetypes(xdg_type)

    copyrights, licenses = parser(fullname)
    logging.debug('Parsed %s: %s, %s', fullname, copyrights, licenses)
    return copyrights, licenses
