#!/usr/bin/env python3
"""Script 54: temporal stability of the BCD inter-CPA floor.
Does the normative BCD per-comparison HC coincidence floor drift over time /
get contaminated by post-2020 e-signing? Compares eras full / 2013-2019 /
2020-2023 using the pool-size-independent per-comparison joint HC ICCR
(cos>0.95 & dHash<=5) on BCD inter-CPA pairs (N=500k, seed 42), plus the
observed deployed per-signature HC rate by firm by era. Read-only.
"""
import sqlite3
from collections import defaultdict
import numpy as np

DB = '/Volumes/NV2/PDF-Processing/signature-analysis/signature_analysis.db'
FIRM_A = '勤業眾信聯合'
BIG4 = ('勤業眾信聯合', '安侯建業聯合', '資誠聯合', '安永聯合')
ALIAS = {'勤業眾信聯合': 'A', '安侯建業聯合': 'B', '資誠聯合': 'C', '安永聯合': 'D'}
SEED = 42
N_PAIRS = 500_000
POP = np.array([bin(i).count('1') for i in range(256)], dtype=np.uint8)


def wilson(k, n, z=1.96):
    if n == 0:
        return (None, None)
    p = k/n; d = 1+z*z/n
    c = (p+z*z/(2*n))/d
    h = z*np.sqrt(p*(1-p)/n+z*z/(4*n*n))/d
    return (max(0.0, c-h), min(1.0, c+h))


conn = sqlite3.connect(f'file:{DB}?mode=ro', uri=True)
cur = conn.cursor()
cur.execute("""
    SELECT s.assigned_accountant, a.firm, CAST(substr(s.year_month,1,4) AS INT),
           s.feature_vector, s.dhash_vector,
           s.max_similarity_to_same_accountant, s.min_dhash_independent
    FROM signatures s JOIN accountants a ON s.assigned_accountant=a.name
    WHERE a.firm IN (?,?,?,?) AND s.year_month IS NOT NULL
      AND s.feature_vector IS NOT NULL AND s.dhash_vector IS NOT NULL""", BIG4)
rows = cur.fetchall()
conn.close()

ERAS = {'full 2013-2023': lambda y: True,
        '2013-2019 (pre-drift)': lambda y: 2013 <= y <= 2019,
        '2020-2023': lambda y: 2020 <= y <= 2023}


def per_comparison_floor(era_fn, label):
    # BCD-only (exclude Firm A), era-restricted
    keep = [r for r in rows if r[1] != FIRM_A and era_fn(r[2])]
    feats = np.stack([np.frombuffer(r[3], np.float32) for r in keep]).astype(np.float32)
    feats /= np.clip(np.linalg.norm(feats, axis=1, keepdims=True), 1e-9, None)
    dh = np.stack([np.frombuffer(r[4], np.uint8) for r in keep])
    cpas = np.array([r[0] for r in keep])
    by = defaultdict(list)
    for i, c in enumerate(cpas):
        by[c].append(i)
    accts = list(by.keys())
    rng = np.random.default_rng(SEED)
    cos = np.empty(N_PAIRS, np.float32); dv = np.empty(N_PAIRS, np.int32)
    na = len(accts)
    for t in range(N_PAIRS):
        i, j = rng.choice(na, 2, replace=False)
        a1, a2 = accts[i], accts[j]
        k1 = by[a1][int(rng.integers(0, len(by[a1])))]
        k2 = by[a2][int(rng.integers(0, len(by[a2])))]
        cos[t] = feats[k1] @ feats[k2]
        dv[t] = POP[dh[k1] ^ dh[k2]].sum()
    joint = int(((cos > 0.95) & (dv <= 5)).sum())
    lo, hi = wilson(joint, N_PAIRS)
    print(f'  [{label}] BCD per-comparison HC floor = {joint/N_PAIRS:.6f} '
          f'({joint}/{N_PAIRS}) Wilson95% [{lo:.6f},{hi:.6f}]  '
          f'(n_sig={len(keep):,}, CPAs={na})')
    return joint/N_PAIRS


print('=== (1) BCD per-comparison HC floor by era (pool-size-independent) ===')
floors = {lab: per_comparison_floor(fn, lab) for lab, fn in ERAS.items()}

print('\n=== (2) Observed deployed per-signature HC rate by firm by era ===')
print('   (max_sim>0.95 & min_dh<=5 on actual same-CPA pools)')
for lab, fn in ERAS.items():
    print(f'  --- {lab} ---')
    for fm_zh in BIG4:
        sub = [r for r in rows if r[1] == fm_zh and fn(r[2])
               and r[5] is not None and r[6] is not None]
        if not sub:
            continue
        k = sum(1 for r in sub if r[5] > 0.95 and r[6] <= 5)
        print(f'      Firm {ALIAS[fm_zh]}: {k/len(sub):.4f} ({k}/{len(sub)})')

print('\n=== A-vs-floor multiple by era (observed A HC / BCD floor) ===')
for lab, fn in ERAS.items():
    a = [r for r in rows if r[1] == FIRM_A and fn(r[2]) and r[5] is not None and r[6] is not None]
    a_rate = sum(1 for r in a if r[5] > 0.95 and r[6] <= 5)/len(a) if a else 0
    fl = floors[lab]
    # per-comparison floor is not directly comparable to observed pooled rate;
    # report ratio vs the per-signature floor proxy from Script 52 (0.0116 full).
    print(f'  {lab}: observed A HC = {a_rate:.3f}; per-comparison floor = {fl:.6f}')
