#!/usr/bin/env python3
"""
Haiti OSINT Document Parser — PII, Credentials, and Sensitive Data Extraction
Scans PDF, DOCX, DOC, XLSX files from DINEPA, MICT, DOUANE dumps.
Also checks EXIF GPS data on images from MICT-GOUV/uploads/.

Uses subprocess isolation per-file to survive segfaults from corrupt PDFs.
"""

import os
import re
import sys
import json
import time
import signal
import subprocess
import traceback
from pathlib import Path
from collections import defaultdict
from datetime import datetime

# ─── Configuration ──────────────────────────────────────────────────
BASE = Path(r"C:\Users\Squir\Desktop\HAITI\DUMP")

SCAN_DIRS = [
    BASE / "DINEPA-GOUV" / "downloads",
    BASE / "MICT-GOUV",
    BASE / "DOUANE-GOUV" / "downloads",
]

IMAGE_DIRS = [
    BASE / "MICT-GOUV" / "uploads",
]

DOC_EXTENSIONS = {".pdf", ".docx", ".doc", ".xlsx"}
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tiff", ".tif", ".gif", ".bmp"}

OUTPUT_REPORT  = BASE / "DOCUMENT-PII-REPORT.md"
OUTPUT_EMAILS  = BASE / "DOCUMENT-EMAILS.txt"
OUTPUT_PHONES  = BASE / "DOCUMENT-PHONES.txt"
OUTPUT_EXIF    = BASE / "EXIF-REPORT.txt"

# ─── Regex patterns ────────────────────────────────────────────────
PATTERNS_RAW = {
    "email": (r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}', 0),
    "phone_haiti": (r'(?:\+509|509)[\s.\-]?\d{4}[\s.\-]?\d{4}', 0),
    "phone_general": (r'(?:\+\d{1,3}[\s.\-])?\(?\d{2,4}\)?[\s.\-]?\d{3,4}[\s.\-]?\d{3,4}', 0),
    "nif_tax": (r'\d{3}-\d{3}-\d{3}-\d{1}', 0),
    "cin_id": (r'(?:CIN|NIF|NIN)[:\s]*[\d\-]+', 0),
    "address": (r'(?:Rue|Avenue|Route|Boulevard|Blvd|Impasse|Ruelle)\s+[A-Z\u00C0-\u00DC][a-z\u00E0-\u00FC\s,.\-]+(?:Port-au-Prince|P[eé]tion-Ville|Delmas|Carrefour|Cap-Ha[iï]tien)?', 0),
    "named_person": (r'(?:Monsieur|Madame|M\.|Mme|Dr\.?|Ing\.?|Prof\.?)\s+[A-Z\u00C0-\u00DC][a-z\u00E0-\u00FC]+\s+[A-Z\u00C0-\u00DC][A-Z\u00C0-\u00DCa-z\u00E0-\u00FC]+', 0),
    "url": (r'https?://[^\s<>"\']+', 0),
    "credential": (r'(?:password|passwd|pwd|mot de passe|secret|token|api.?key)[:\s=]+\S+', re.IGNORECASE),
    "db_string": (r'(?:mysql|postgres|mongodb|redis)://[^\s<>"\']+', re.IGNORECASE),
    "ip_address": (r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b', 0),
}

EMAIL_JUNK = re.compile(r'(?:\.png|\.jpg|\.gif|\.svg|\.css|\.js|\.woff|\.ttf|\.pdf|\.docx|\.xlsx|\.php|\.html)$', re.IGNORECASE)
PHONE_JUNK = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$')

# ─── WORKER SCRIPT (runs in subprocess per file) ───────────────────
WORKER_SCRIPT = r'''
import sys, json, re, os

filepath = sys.argv[1]
ext = os.path.splitext(filepath)[1].lower()

results = []  # list of (location, text_chunk)

try:
    if ext == ".pdf":
        import pdfplumber
        with pdfplumber.open(filepath) as pdf:
            for i, page in enumerate(pdf.pages, 1):
                try:
                    text = page.extract_text()
                    if text:
                        results.append((f"page {i}", text))
                    tables = page.extract_tables()
                    if tables:
                        for table in tables:
                            for row in table:
                                if row:
                                    row_text = " | ".join(str(c) if c else "" for c in row)
                                    results.append((f"page {i} (table)", row_text))
                except Exception as e:
                    results.append((f"page {i} ERROR", str(e)))

    elif ext == ".docx":
        from docx import Document as DocxDocument
        doc = DocxDocument(filepath)
        parts = []
        for para in doc.paragraphs:
            if para.text.strip():
                parts.append(para.text)
        for table in doc.tables:
            for row in table.rows:
                row_text = " | ".join(cell.text for cell in row.cells)
                parts.append(row_text)
        if parts:
            results.append(("full document", "\n".join(parts)))

    elif ext == ".xlsx":
        import openpyxl
        wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
        for sheet_name in wb.sheetnames:
            ws = wb[sheet_name]
            for row_idx, row in enumerate(ws.iter_rows(values_only=True), 1):
                if row:
                    row_text = " | ".join(str(c) if c is not None else "" for c in row)
                    results.append((f"sheet '{sheet_name}' row {row_idx}", row_text))
        wb.close()

    elif ext == ".doc":
        with open(filepath, "rb") as f:
            data = f.read()
        ascii_strings = re.findall(rb'[\x20-\x7E]{6,}', data)
        combined = "\n".join(s.decode('ascii', errors='ignore') for s in ascii_strings)
        try:
            utf16 = data.decode('utf-16-le', errors='ignore')
            printable = re.findall(r'[\x20-\x7E]{6,}', utf16)
            combined += "\n" + "\n".join(printable)
        except:
            pass
        if combined.strip():
            results.append(("binary extraction", combined))

    # Output as JSON
    print(json.dumps({"status": "ok", "results": results}))

except Exception as e:
    print(json.dumps({"status": "error", "error": str(e)}))
'''


def relpath(p):
    try:
        return str(Path(p).relative_to(BASE))
    except ValueError:
        return str(p)


def parse_file_subprocess(filepath, timeout=120):
    """Parse a single file in a subprocess. Returns (status, results_or_error)."""
    try:
        proc = subprocess.run(
            [sys.executable, "-c", WORKER_SCRIPT, str(filepath)],
            capture_output=True, text=True, timeout=timeout,
            cwd=str(BASE)
        )
        stdout = proc.stdout.strip()
        if not stdout:
            if proc.returncode != 0:
                return ("error", f"Exit code {proc.returncode}: {proc.stderr[:300]}")
            return ("error", "No output from worker")

        # Find the JSON line (last line that starts with {)
        for line in reversed(stdout.split("\n")):
            line = line.strip()
            if line.startswith("{"):
                data = json.loads(line)
                return (data.get("status", "error"), data.get("results", data.get("error", "unknown")))

        return ("error", f"No JSON in output: {stdout[:300]}")

    except subprocess.TimeoutExpired:
        return ("error", f"Timeout after {timeout}s")
    except json.JSONDecodeError as e:
        return ("error", f"JSON decode error: {e}, output: {stdout[:300]}")
    except Exception as e:
        return ("error", str(e))


def scan_text_for_matches(text, source_file, location):
    """Run all regex patterns on text. Returns list of (pattern_name, value)."""
    matches = []
    if not text or not text.strip():
        return matches
    for name, (pattern_str, flags) in PATTERNS_RAW.items():
        pat = re.compile(pattern_str, flags)
        for m in pat.finditer(text):
            val = m.group(0).strip()
            if name == "email" and (EMAIL_JUNK.search(val) or len(val) < 6):
                continue
            if name in ("phone_haiti", "phone_general") and (PHONE_JUNK.match(val) or len(val) < 7):
                continue
            matches.append((name, location, val))
    return matches


def scan_exif_batch(image_files):
    """Scan images for EXIF data. Returns list of (relpath, data_dict)."""
    results = []
    import exifread
    for fp in image_files:
        try:
            with open(fp, "rb") as f:
                tags = exifread.process_file(f, details=False)
            gps_keys = [k for k in tags if "GPS" in k]
            if gps_keys:
                gps_data = {k: str(tags[k]) for k in gps_keys}
                results.append((relpath(fp), gps_data))
                continue
            # Check for author/metadata
            interesting = {}
            for key in ["Image Software", "Image Make", "Image Model",
                        "EXIF DateTimeOriginal", "EXIF DateTimeDigitized",
                        "Image Artist", "Image Copyright", "Image XPAuthor",
                        "Image XPComment"]:
                if key in tags:
                    interesting[key] = str(tags[key])
            author_keys = [k for k in interesting if "Author" in k or "Artist" in k or "Comment" in k or "Copyright" in k]
            if author_keys:
                results.append((relpath(fp), interesting))
        except:
            pass
    return results


def main():
    start = time.time()
    print("=" * 70)
    print("HAITI OSINT DOCUMENT PARSER — PII & CREDENTIAL EXTRACTION")
    print(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("=" * 70)

    # ── Collect document files ──────────────────────────────────────
    doc_files = []
    for scan_dir in SCAN_DIRS:
        if not scan_dir.exists():
            print(f"[WARN] Directory not found: {scan_dir}")
            continue
        for root, dirs, files in os.walk(scan_dir):
            for fname in files:
                fp = Path(root) / fname
                if fp.suffix.lower() in DOC_EXTENSIONS:
                    doc_files.append(fp)

    # Deduplicate
    seen = set()
    unique_docs = []
    for fp in doc_files:
        resolved = str(fp.resolve())
        if resolved not in seen:
            seen.add(resolved)
            unique_docs.append(fp)
    doc_files = unique_docs

    print(f"\n[*] Found {len(doc_files)} document files to parse")
    by_ext = defaultdict(int)
    for f in doc_files:
        by_ext[f.suffix.lower()] += 1
    for ext, cnt in sorted(by_ext.items()):
        print(f"    {ext}: {cnt}")

    # ── Parse documents (subprocess per file) ───────────────────────
    findings = defaultdict(list)  # pattern_name -> [(file, location, value)]
    all_emails = set()
    all_phones = set()
    processed = 0
    errors = 0
    error_log = []

    print(f"\n[*] Parsing documents (subprocess isolation)...")
    for i, fp in enumerate(doc_files, 1):
        rp = relpath(fp)
        print(f"  [{i}/{len(doc_files)}] {rp}", end=" ... ", flush=True)

        status, result = parse_file_subprocess(fp, timeout=180)

        if status == "error":
            errors += 1
            error_log.append((rp, str(result)[:300]))
            print(f"ERROR: {str(result)[:80]}")
            continue

        processed += 1
        match_count = 0
        for location, text_chunk in result:
            if "ERROR" in location:
                continue
            matches = scan_text_for_matches(text_chunk, fp, location)
            for name, loc, val in matches:
                findings[name].append((rp, loc, val))
                if name == "email":
                    all_emails.add(val)
                if name in ("phone_haiti", "phone_general"):
                    all_phones.add(val)
                match_count += 1
        print(f"OK ({match_count} matches)")

    # ── EXIF scan ───────────────────────────────────────────────────
    image_files = []
    for img_dir in IMAGE_DIRS:
        if not img_dir.exists():
            continue
        for root, dirs, files in os.walk(img_dir):
            for fname in files:
                fp = Path(root) / fname
                if fp.suffix.lower() in IMAGE_EXTENSIONS:
                    image_files.append(fp)

    print(f"\n[*] Scanning {len(image_files)} images for EXIF GPS data...")
    exif_results = scan_exif_batch(image_files)
    gps_count = sum(1 for _, d in exif_results if any("GPS" in k for k in d))
    meta_count = len(exif_results) - gps_count
    print(f"    GPS locations found: {gps_count}")
    print(f"    Author/metadata found: {meta_count}")

    elapsed = time.time() - start

    # ── Generate reports ────────────────────────────────────────────
    print(f"\n[*] Generating reports...")

    category_labels = {
        "email": "Email Addresses",
        "phone_haiti": "Haiti Phone Numbers (+509)",
        "phone_general": "Phone Numbers (General)",
        "nif_tax": "NIF Tax IDs",
        "cin_id": "CIN/NIF/NIN IDs",
        "address": "Physical Addresses",
        "named_person": "Named Individuals",
        "url": "URLs",
        "credential": "Credentials/Passwords",
        "db_string": "Database Connection Strings",
        "ip_address": "IP Addresses",
    }

    # ── Main PII Report ─────────────────────────────────────────────
    with open(OUTPUT_REPORT, "w", encoding="utf-8") as rpt:
        rpt.write("# Haiti OSINT Document PII & Credential Report\n\n")
        rpt.write(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}  \n")
        rpt.write(f"**Documents parsed:** {processed}  \n")
        rpt.write(f"**Parse errors:** {errors}  \n")
        rpt.write(f"**Images EXIF-scanned:** {len(image_files)}  \n")
        rpt.write(f"**Runtime:** {elapsed:.1f}s  \n\n")

        rpt.write("---\n\n## Summary of Findings\n\n")
        rpt.write("| Category | Matches | Unique Values |\n")
        rpt.write("|----------|---------|---------------|\n")
        for key in category_labels:
            matches = findings.get(key, [])
            unique_vals = set(m[2] for m in matches)
            rpt.write(f"| {category_labels[key]} | {len(matches)} | {len(unique_vals)} |\n")

        rpt.write(f"\n**EXIF GPS locations found:** {gps_count}  \n")
        rpt.write(f"**EXIF author/metadata found:** {meta_count}  \n\n")

        # ── Detailed findings ───────────────────────────────────────
        rpt.write("---\n\n## Detailed Findings\n\n")

        for key in category_labels:
            matches = findings.get(key, [])
            if not matches:
                continue
            unique_vals = set(m[2] for m in matches)
            rpt.write(f"### {category_labels[key]} ({len(matches)} matches, {len(unique_vals)} unique)\n\n")

            by_file = defaultdict(list)
            for f, loc, val in matches:
                by_file[f].append((loc, val))

            for fname in sorted(by_file.keys()):
                rpt.write(f"**{fname}**\n")
                seen_vals = set()
                for loc, val in by_file[fname]:
                    display = val[:250]
                    if display not in seen_vals:
                        rpt.write(f"- `{display}` ({loc})\n")
                        seen_vals.add(display)
                rpt.write("\n")

        # ── High-value findings section ─────────────────────────────
        rpt.write("---\n\n## HIGH-VALUE FINDINGS\n\n")

        # Credentials
        creds = findings.get("credential", [])
        if creds:
            rpt.write("### Credentials / Passwords\n\n")
            for f, loc, val in creds:
                rpt.write(f"- **{f}** ({loc}): `{val[:200]}`\n")
            rpt.write("\n")

        # DB strings
        dbs = findings.get("db_string", [])
        if dbs:
            rpt.write("### Database Connection Strings\n\n")
            for f, loc, val in dbs:
                rpt.write(f"- **{f}** ({loc}): `{val[:200]}`\n")
            rpt.write("\n")

        # NIF Tax IDs
        nifs = findings.get("nif_tax", [])
        if nifs:
            rpt.write("### NIF Tax IDs\n\n")
            for f, loc, val in nifs:
                rpt.write(f"- **{f}** ({loc}): `{val}`\n")
            rpt.write("\n")

        # CIN IDs
        cins = findings.get("cin_id", [])
        if cins:
            rpt.write("### CIN/NIF/NIN National IDs\n\n")
            for f, loc, val in cins:
                rpt.write(f"- **{f}** ({loc}): `{val}`\n")
            rpt.write("\n")

        # Named Persons (first 100)
        persons = findings.get("named_person", [])
        if persons:
            rpt.write(f"### Named Individuals (showing up to 100 of {len(persons)})\n\n")
            unique_persons = set()
            count = 0
            for f, loc, val in persons:
                if val not in unique_persons and count < 100:
                    rpt.write(f"- `{val}` — {f} ({loc})\n")
                    unique_persons.add(val)
                    count += 1
            rpt.write("\n")

        # ── Error log ───────────────────────────────────────────────
        if error_log:
            rpt.write("---\n\n## Parse Errors\n\n")
            for fname, err in error_log:
                rpt.write(f"- **{fname}**: {err[:200]}\n")
            rpt.write("\n")

        # ── EXIF section ────────────────────────────────────────────
        if exif_results:
            rpt.write("---\n\n## EXIF Metadata Findings\n\n")
            for fname, data in exif_results:
                rpt.write(f"**{fname}**\n")
                for k, v in data.items():
                    rpt.write(f"- {k}: `{v}`\n")
                rpt.write("\n")

    # ── Emails file ─────────────────────────────────────────────────
    with open(OUTPUT_EMAILS, "w", encoding="utf-8") as f:
        f.write(f"# Haiti OSINT Document Email Extraction\n")
        f.write(f"# Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        f.write(f"# Total unique emails: {len(all_emails)}\n\n")
        for email in sorted(all_emails, key=str.lower):
            f.write(email + "\n")

    # ── Phones file ─────────────────────────────────────────────────
    with open(OUTPUT_PHONES, "w", encoding="utf-8") as f:
        f.write(f"# Haiti OSINT Document Phone Extraction\n")
        f.write(f"# Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        f.write(f"# Total unique phones: {len(all_phones)}\n\n")
        for phone in sorted(all_phones):
            f.write(phone + "\n")

    # ── EXIF report ─────────────────────────────────────────────────
    with open(OUTPUT_EXIF, "w", encoding="utf-8") as f:
        f.write(f"# Haiti OSINT EXIF GPS & Metadata Report\n")
        f.write(f"# Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        f.write(f"# Images scanned: {len(image_files)}\n")
        f.write(f"# Files with GPS data: {gps_count}\n")
        f.write(f"# Files with author/metadata: {meta_count}\n\n")
        if exif_results:
            for fname, data in exif_results:
                f.write(f"FILE: {fname}\n")
                for k, v in data.items():
                    f.write(f"  {k}: {v}\n")
                f.write("\n")
        else:
            f.write("No EXIF GPS or notable metadata found.\n")

    # ── Console summary ─────────────────────────────────────────────
    print("\n" + "=" * 70)
    print("RESULTS SUMMARY")
    print("=" * 70)
    print(f"Documents processed: {processed}")
    print(f"Parse errors:        {errors}")
    print(f"Images EXIF-scanned: {len(image_files)}")
    print(f"Runtime:             {elapsed:.1f}s")
    print()
    total_matches = 0
    for key in category_labels:
        matches = findings.get(key, [])
        unique_vals = set(m[2] for m in matches)
        total_matches += len(matches)
        if matches:
            print(f"  {category_labels[key]:35s}  {len(matches):6d} matches  ({len(unique_vals)} unique)")
    print(f"\n  TOTAL MATCHES:                       {total_matches:6d}")
    print(f"  Unique emails extracted:              {len(all_emails):6d}")
    print(f"  Unique phones extracted:              {len(all_phones):6d}")
    print(f"  EXIF GPS locations:                   {gps_count:6d}")
    print()
    print(f"Reports written to:")
    print(f"  {OUTPUT_REPORT}")
    print(f"  {OUTPUT_EMAILS}")
    print(f"  {OUTPUT_PHONES}")
    print(f"  {OUTPUT_EXIF}")
    print("=" * 70)


if __name__ == "__main__":
    main()