#!/usr/bin/env python3
"""Deduplication — Appendix E, specification 4.

Takes:  a directory of files, or a CSV with a key column.
Emits:  the GROUPS, not just the survivors, so that the decision to treat two items
        as the same item is visible and reviewable.

Nothing is deleted. Standard library only. Deterministic.

  python3 deduplicate.py --dir ./production --by content --out dupes.json
  python3 deduplicate.py --csv rows.csv --col Email --by value --normalize --out dupes.json
"""
import argparse, csv, hashlib, json, os, platform, sys
from datetime import datetime, timezone

VERSION = "1.0"

def sha256_file(p):
    h = hashlib.sha256()
    with open(p, "rb") as f:
        for b in iter(lambda: f.read(1 << 20), b""):
            h.update(b)
    return h.hexdigest()

def main():
    ap = argparse.ArgumentParser(description="Group duplicates and show the groups.")
    src = ap.add_mutually_exclusive_group(required=True)
    src.add_argument("--dir"); src.add_argument("--csv")
    ap.add_argument("--col", help="key column (required with --csv)")
    ap.add_argument("--by", choices=["content", "name", "value"], default="content",
                    help="content = SHA-256 of bytes; name = basename; value = the column value")
    ap.add_argument("--normalize", action="store_true", help="trim and casefold the key before grouping")
    ap.add_argument("--out", required=True)
    a = ap.parse_args()

    groups, unreadable, read = {}, [], 0
    if a.dir:
        if a.by == "value":
            sys.exit("--by value applies to --csv; use content or name with --dir")
        for dp, dns, fns in os.walk(a.dir):
            dns.sort()
            for fn in sorted(fns):
                p = os.path.join(dp, fn)
                rel = os.path.relpath(p, a.dir).replace(os.sep, "/")
                read += 1
                try:
                    key = sha256_file(p) if a.by == "content" else fn
                except OSError as e:
                    unreadable.append({"item": rel, "error": str(e)}); continue
                if a.normalize and a.by == "name":
                    key = key.strip().casefold()
                groups.setdefault(key, []).append(rel)
    else:
        if not a.col:
            sys.exit("--csv requires --col")
        with open(a.csv, newline="", encoding="utf-8-sig", errors="replace") as f:
            r = csv.DictReader(f)
            if r.fieldnames is None or a.col not in r.fieldnames:
                sys.exit(f"{a.csv}: column {a.col!r} not found; columns are {r.fieldnames}")
            for i, row in enumerate(r, start=2):
                read += 1
                v = (row.get(a.col) or "").strip()
                if not v:
                    unreadable.append({"row": i, "error": "empty key"}); continue
                key = v.casefold() if a.normalize else v
                groups.setdefault(key, []).append(f"row {i}: {v}")

    dupe_groups = {k: sorted(v) for k, v in groups.items() if len(v) > 1}
    singles = sum(1 for v in groups.values() if len(v) == 1)
    redundant = sum(len(v) - 1 for v in dupe_groups.values())

    report = {
        "specification": "Deduplication (Appendix E, 4)",
        "script": os.path.basename(__file__), "script_version": VERSION,
        "run_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "python": platform.python_version(),
        "parameters": {"dir": a.dir, "csv": a.csv, "col": a.col, "by": a.by,
                       "normalize": a.normalize},
        "totals": {"items_read": read, "distinct_keys": len(groups),
                   "groups_with_duplicates": len(dupe_groups),
                   "items_that_are_redundant_copies": redundant,
                   "unique_items": singles, "unreadable": len(unreadable)},
        "duplicate_groups": [{"key": k, "members": dupe_groups[k]} for k in sorted(dupe_groups)],
        "unreadable": unreadable,
        "note": "Nothing was deleted. Each group is shown so the grouping decision can be reviewed.",
    }
    with open(a.out, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=1); f.write("\n")
    t = report["totals"]
    print(f"read={t['items_read']} groups_with_duplicates={t['groups_with_duplicates']} "
          f"redundant={t['items_that_are_redundant_copies']} unreadable={t['unreadable']} -> {a.out}")

if __name__ == "__main__":
    main()