#!/usr/bin/env python3
"""
Generate PDF-level aggregated report with multi-method verdicts.
One row per PDF with all Group A-F columns plus new SSIM/pHash/combined verdicts.
"""

import sqlite3
import csv
import numpy as np
from datetime import datetime
from collections import defaultdict

DB_PATH = '/Volumes/NV2/PDF-Processing/signature-analysis/signature_analysis.db'
OUTPUT_CSV = '/Volumes/NV2/PDF-Processing/signature-analysis/reports/pdf_level_complete_report.csv'

# Thresholds from statistical analysis
COSINE_THRESHOLD = 0.95
COSINE_STATISTICAL = 0.944  # mu + 2*sigma
KDE_CROSSOVER = 0.838
SSIM_HIGH = 0.95
SSIM_MEDIUM = 0.80
PHASH_IDENTICAL = 0
PHASH_SIMILAR = 5


def classify_overall(max_cosine, max_ssim, min_phash, has_pixel_identical):
    """
    Multi-method combined verdict.
    Returns (verdict, confidence_level, n_methods_agree)
    """
    evidence_copy = 0
    evidence_genuine = 0
    total_methods = 0

    # Method 1: Cosine similarity
    if max_cosine is not None:
        total_methods += 1
        if max_cosine > COSINE_THRESHOLD:
            evidence_copy += 1
        elif max_cosine < KDE_CROSSOVER:
            evidence_genuine += 1

    # Method 2: SSIM
    if max_ssim is not None:
        total_methods += 1
        if max_ssim > SSIM_HIGH:
            evidence_copy += 1
        elif max_ssim < 0.5:
            evidence_genuine += 1

    # Method 3: pHash
    if min_phash is not None:
        total_methods += 1
        if min_phash <= PHASH_IDENTICAL:
            evidence_copy += 1
        elif min_phash > 15:
            evidence_genuine += 1

    # Method 4: Pixel identical
    if has_pixel_identical is not None:
        total_methods += 1
        if has_pixel_identical:
            evidence_copy += 1

    # Decision logic
    if has_pixel_identical:
        verdict = 'definite_copy'
        confidence = 'very_high'
    elif max_ssim is not None and max_ssim > SSIM_HIGH and min_phash is not None and min_phash <= PHASH_SIMILAR:
        verdict = 'definite_copy'
        confidence = 'very_high'
    elif evidence_copy >= 3:
        verdict = 'very_likely_copy'
        confidence = 'high'
    elif evidence_copy >= 2:
        verdict = 'likely_copy'
        confidence = 'medium'
    elif max_cosine is not None and max_cosine > COSINE_THRESHOLD:
        verdict = 'likely_copy'
        confidence = 'medium'
    elif max_cosine is not None and max_cosine > KDE_CROSSOVER:
        verdict = 'uncertain'
        confidence = 'low'
    elif max_cosine is not None and max_cosine <= KDE_CROSSOVER:
        verdict = 'likely_genuine'
        confidence = 'medium'
    else:
        verdict = 'unknown'
        confidence = 'none'

    return verdict, confidence, evidence_copy, total_methods


def main():
    print("=" * 70)
    print("PDF-Level Aggregated Report Generator")
    print("=" * 70)

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    # Load all signature data grouped by PDF
    print("\n[1/3] Loading signature data...")
    cur.execute('''
        SELECT
            s.source_pdf,
            s.year_month,
            s.serial_number,
            s.doc_type,
            s.page_number,
            s.sig_index,
            s.assigned_accountant,
            s.excel_accountant1,
            s.excel_accountant2,
            s.excel_firm,
            s.detection_confidence,
            s.signature_verdict,
            s.max_similarity_to_same_accountant,
            s.ssim_to_closest,
            s.phash_distance_to_closest,
            s.histogram_corr_to_closest,
            s.pixel_identical_to_closest,
            a.risk_level,
            a.mean_similarity,
            a.ratio_gt_95,
            a.signature_count
        FROM signatures s
        LEFT JOIN accountants a ON s.assigned_accountant = a.name
        ORDER BY s.source_pdf, s.sig_index
    ''')

    # Group by PDF
    pdf_data = defaultdict(list)
    for row in cur:
        pdf_data[row[0]].append(row)

    print(f"  {len(pdf_data)} PDFs loaded")

    # Generate PDF-level rows
    print("\n[2/3] Aggregating per-PDF statistics...")

    columns = [
        # Group A: PDF Identity
        'source_pdf', 'year_month', 'serial_number', 'doc_type',

        # Group B: Excel Master Data
        'accountant_1', 'accountant_2', 'firm',

        # Group C: YOLO Detection
        'n_signatures_detected', 'avg_detection_confidence',

        # Group D: Cosine Similarity
        'max_cosine_similarity', 'min_cosine_similarity', 'avg_cosine_similarity',

        # Group E: Verdict (original per-sig)
        'sig1_cosine_verdict', 'sig2_cosine_verdict',

        # Group F: Accountant Risk
        'acct1_name', 'acct1_risk_level', 'acct1_mean_similarity',
        'acct1_ratio_gt_95', 'acct1_total_signatures',
        'acct2_name', 'acct2_risk_level', 'acct2_mean_similarity',
        'acct2_ratio_gt_95', 'acct2_total_signatures',

        # Group G: SSIM (NEW)
        'max_ssim', 'min_ssim', 'avg_ssim',
        'verdict_ssim',

        # Group H: pHash (NEW)
        'min_phash_distance', 'max_phash_distance', 'avg_phash_distance',
        'verdict_phash',

        # Group I: Histogram Correlation (NEW)
        'max_histogram_corr', 'avg_histogram_corr',

        # Group J: Pixel Identity (NEW)
        'has_pixel_identical',
        'verdict_pixel',

        # Group K: Statistical Threshold (NEW)
        'verdict_statistical',  # Based on mu+2sigma (0.944)

        # Group L: KDE Crossover (NEW)
        'verdict_kde',  # Based on KDE crossover (0.838)

        # Group M: Multi-Method Combined (NEW)
        'overall_verdict',
        'confidence_level',
        'n_methods_copy',
        'n_methods_total',
    ]

    rows = []
    for pdf_name, sigs in pdf_data.items():
        # Group A: Identity (from first signature)
        first = sigs[0]
        year_month = first[1]
        serial_number = first[2]
        doc_type = first[3]

        # Group B: Excel data
        excel_acct1 = first[7]
        excel_acct2 = first[8]
        excel_firm = first[9]

        # Group C: Detection
        n_sigs = len(sigs)
        confidences = [s[10] for s in sigs if s[10] is not None]
        avg_conf = np.mean(confidences) if confidences else None

        # Group D: Cosine similarity
        cosines = [s[12] for s in sigs if s[12] is not None]
        max_cosine = max(cosines) if cosines else None
        min_cosine = min(cosines) if cosines else None
        avg_cosine = np.mean(cosines) if cosines else None

        # Group E: Per-sig verdicts
        verdicts = [s[11] for s in sigs]
        sig1_verdict = verdicts[0] if len(verdicts) > 0 else None
        sig2_verdict = verdicts[1] if len(verdicts) > 1 else None

        # Group F: Accountant risk - separate for acct1 and acct2
        # Match by assigned_accountant to excel_accountant1/2
        acct1_info = {'name': None, 'risk': None, 'mean_sim': None, 'ratio': None, 'count': None}
        acct2_info = {'name': None, 'risk': None, 'mean_sim': None, 'ratio': None, 'count': None}

        for s in sigs:
            assigned = s[6]
            if assigned and assigned == excel_acct1 and acct1_info['name'] is None:
                acct1_info = {
                    'name': assigned, 'risk': s[17],
                    'mean_sim': s[18], 'ratio': s[19], 'count': s[20]
                }
            elif assigned and assigned == excel_acct2 and acct2_info['name'] is None:
                acct2_info = {
                    'name': assigned, 'risk': s[17],
                    'mean_sim': s[18], 'ratio': s[19], 'count': s[20]
                }
            elif assigned and acct1_info['name'] is None:
                acct1_info = {
                    'name': assigned, 'risk': s[17],
                    'mean_sim': s[18], 'ratio': s[19], 'count': s[20]
                }
            elif assigned and acct2_info['name'] is None:
                acct2_info = {
                    'name': assigned, 'risk': s[17],
                    'mean_sim': s[18], 'ratio': s[19], 'count': s[20]
                }

        # Group G: SSIM
        ssims = [s[13] for s in sigs if s[13] is not None]
        max_ssim = max(ssims) if ssims else None
        min_ssim = min(ssims) if ssims else None
        avg_ssim = np.mean(ssims) if ssims else None

        if max_ssim is not None:
            if max_ssim > SSIM_HIGH:
                verdict_ssim = 'copy'
            elif max_ssim > SSIM_MEDIUM:
                verdict_ssim = 'suspicious'
            else:
                verdict_ssim = 'genuine'
        else:
            verdict_ssim = None

        # Group H: pHash
        phashes = [s[14] for s in sigs if s[14] is not None]
        min_phash = min(phashes) if phashes else None
        max_phash = max(phashes) if phashes else None
        avg_phash = np.mean(phashes) if phashes else None

        if min_phash is not None:
            if min_phash <= PHASH_IDENTICAL:
                verdict_phash = 'copy'
            elif min_phash <= PHASH_SIMILAR:
                verdict_phash = 'suspicious'
            else:
                verdict_phash = 'genuine'
        else:
            verdict_phash = None

        # Group I: Histogram correlation
        histcorrs = [s[15] for s in sigs if s[15] is not None]
        max_histcorr = max(histcorrs) if histcorrs else None
        avg_histcorr = np.mean(histcorrs) if histcorrs else None

        # Group J: Pixel identical
        pixel_ids = [s[16] for s in sigs if s[16] is not None]
        has_pixel = any(p == 1 for p in pixel_ids) if pixel_ids else False
        verdict_pixel = 'copy' if has_pixel else 'genuine'

        # Group K: Statistical threshold (mu+2sigma = 0.944)
        if max_cosine is not None:
            if max_cosine > COSINE_STATISTICAL:
                verdict_stat = 'copy'
            elif max_cosine > KDE_CROSSOVER:
                verdict_stat = 'uncertain'
            else:
                verdict_stat = 'genuine'
        else:
            verdict_stat = None

        # Group L: KDE crossover (0.838)
        if max_cosine is not None:
            if max_cosine > KDE_CROSSOVER:
                verdict_kde = 'above_crossover'
            else:
                verdict_kde = 'below_crossover'
        else:
            verdict_kde = None

        # Group M: Multi-method combined
        overall, confidence, n_copy, n_total = classify_overall(
            max_cosine, max_ssim, min_phash, has_pixel)

        rows.append([
            # A
            pdf_name, year_month, serial_number, doc_type,
            # B
            excel_acct1, excel_acct2, excel_firm,
            # C
            n_sigs, avg_conf,
            # D
            max_cosine, min_cosine, avg_cosine,
            # E
            sig1_verdict, sig2_verdict,
            # F
            acct1_info['name'], acct1_info['risk'], acct1_info['mean_sim'],
            acct1_info['ratio'], acct1_info['count'],
            acct2_info['name'], acct2_info['risk'], acct2_info['mean_sim'],
            acct2_info['ratio'], acct2_info['count'],
            # G
            max_ssim, min_ssim, avg_ssim, verdict_ssim,
            # H
            min_phash, max_phash, avg_phash, verdict_phash,
            # I
            max_histcorr, avg_histcorr,
            # J
            1 if has_pixel else 0, verdict_pixel,
            # K
            verdict_stat,
            # L
            verdict_kde,
            # M
            overall, confidence, n_copy, n_total,
        ])

    # Write CSV
    print(f"\n[3/3] Writing {len(rows)} PDF rows to CSV...")
    with open(OUTPUT_CSV, 'w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(columns)
        writer.writerows(rows)

    conn.close()

    # Print summary statistics
    print(f"\n{'='*70}")
    print("SUMMARY")
    print(f"{'='*70}")
    print(f"Total PDFs: {len(rows):,}")

    # Overall verdict distribution
    verdict_counts = defaultdict(int)
    confidence_counts = defaultdict(int)
    for r in rows:
        verdict_counts[r[-4]] += 1
        confidence_counts[r[-3]] += 1

    print(f"\n--- Overall Verdict Distribution ---")
    for v in ['definite_copy', 'very_likely_copy', 'likely_copy', 'uncertain', 'likely_genuine', 'unknown']:
        c = verdict_counts.get(v, 0)
        print(f"  {v:20s}: {c:>6,} ({100*c/len(rows):5.1f}%)")

    print(f"\n--- Confidence Level Distribution ---")
    for c_level in ['very_high', 'high', 'medium', 'low', 'none']:
        c = confidence_counts.get(c_level, 0)
        print(f"  {c_level:10s}: {c:>6,} ({100*c/len(rows):5.1f}%)")

    # Per-method verdict distribution
    # Column indices: verdict_ssim=27, verdict_phash=31, verdict_pixel=35, verdict_stat=36, verdict_kde=37
    print(f"\n--- Per-Method Verdict Distribution ---")
    for col_idx, method_name in [(27, 'SSIM'), (31, 'pHash'), (35, 'Pixel'), (36, 'Statistical'), (37, 'KDE')]:
        counts = defaultdict(int)
        for r in rows:
            counts[r[col_idx]] += 1
        print(f"\n  {method_name}:")
        for k, v in sorted(counts.items(), key=lambda x: -x[1]):
            print(f"    {str(k):20s}: {v:>6,} ({100*v/len(rows):5.1f}%)")

    # Cross-method agreement
    print(f"\n--- Method Agreement (cosine>0.95 PDFs) ---")
    cosine_copy = [r for r in rows if r[9] is not None and r[9] > COSINE_THRESHOLD]
    if cosine_copy:
        ssim_agree = sum(1 for r in cosine_copy if r[27] == 'copy')
        phash_agree = sum(1 for r in cosine_copy if r[31] == 'copy')
        pixel_agree = sum(1 for r in cosine_copy if r[34] == 1)
        print(f"  PDFs with cosine > 0.95: {len(cosine_copy):,}")
        print(f"  Also SSIM > 0.95:  {ssim_agree:>6,} ({100*ssim_agree/len(cosine_copy):5.1f}%)")
        print(f"  Also pHash = 0:    {phash_agree:>6,} ({100*phash_agree/len(cosine_copy):5.1f}%)")
        print(f"  Also pixel-identical: {pixel_agree:>4,} ({100*pixel_agree/len(cosine_copy):5.1f}%)")

    print(f"\nOutput: {OUTPUT_CSV}")
    print(f"{'='*70}")


if __name__ == '__main__':
    main()
