#!/usr/bin/env python3
"""Search with coverage — Appendix E, specification 5.

The one most often done badly. Emits not only the hits but the DENOMINATOR: how many
items were searched, how many could not be read, and what was skipped. A search that
silently fails to open forty documents reports zero hits in exactly the same way as a
search that found none.

Searches the extractable text of .txt, .csv, .md, .json, .htm/.html and .xml files.
Binary and unsupported formats are reported as NOT SEARCHED rather than as zero hits.

  python3 search_coverage.py --dir ./production --term "0% at fault" --term prompt --out search.json
"""
import argparse, json, os, platform, re, sys
from datetime import datetime, timezone

VERSION = "1.0"
TEXTLIKE = {".txt", ".csv", ".tsv", ".md", ".json", ".htm", ".html", ".xml", ".log", ".eml"}

def strip_markup(s, ext):
    return re.sub(r"<[^>]+>", " ", s) if ext in {".htm", ".html", ".xml"} else s

def main():
    ap = argparse.ArgumentParser(description="Search a corpus and report coverage as well as hits.")
    ap.add_argument("--dir", required=True)
    ap.add_argument("--term", action="append", required=True, help="search term (repeatable)")
    ap.add_argument("--regex", action="store_true", help="treat terms as regular expressions")
    ap.add_argument("--case-sensitive", action="store_true")
    ap.add_argument("--out", required=True)
    a = ap.parse_args()

    if not os.path.isdir(a.dir):
        sys.exit(f"not a directory: {a.dir}")
    flags = 0 if a.case_sensitive else re.IGNORECASE
    pats = [(t, re.compile(t if a.regex else re.escape(t), flags)) for t in a.term]

    searched, not_searched, unreadable, hits = [], [], [], []
    total_seen = 0
    for dp, dns, fns in os.walk(a.dir):
        dns.sort()
        for fn in sorted(fns):
            p = os.path.join(dp, fn)
            rel = os.path.relpath(p, a.dir).replace(os.sep, "/")
            total_seen += 1
            ext = os.path.splitext(fn)[1].lower()
            if ext not in TEXTLIKE:
                not_searched.append({"item": rel, "reason": f"unsupported format {ext or '(none)'}"})
                continue
            try:
                with open(p, encoding="utf-8", errors="strict") as f:
                    text = f.read()
            except (UnicodeDecodeError, OSError) as e:
                unreadable.append({"item": rel, "error": type(e).__name__})
                continue
            text = strip_markup(text, ext)
            searched.append(rel)
            for term, rx in pats:
                n = len(rx.findall(text))
                if n:
                    m = rx.search(text)
                    s, e = max(0, m.start() - 60), min(len(text), m.end() + 60)
                    hits.append({"item": rel, "term": term, "occurrences": n,
                                 "first_context": " ".join(text[s:e].split())})

    hits.sort(key=lambda h: (h["item"], h["term"]))
    per_term = {t: sum(h["occurrences"] for h in hits if h["term"] == t) for t in a.term}
    per_term_docs = {t: len({h["item"] for h in hits if h["term"] == t}) for t in a.term}

    report = {
        "specification": "Search with coverage (Appendix E, 5)",
        "script": os.path.basename(__file__), "script_version": VERSION,
        "run_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "python": platform.python_version(),
        "parameters": {"dir": a.dir, "terms": a.term, "regex": a.regex,
                       "case_sensitive": a.case_sensitive,
                       "formats_searched": sorted(TEXTLIKE)},
        "coverage": {
            "items_encountered": total_seen,
            "items_searched": len(searched),
            "items_not_searched_unsupported_format": len(not_searched),
            "items_unreadable": len(unreadable),
            "coverage_percent": round(100.0 * len(searched) / total_seen, 2) if total_seen else 0.0,
        },
        "results": {"documents_with_at_least_one_hit": len({h["item"] for h in hits}),
                    "occurrences_by_term": per_term, "documents_by_term": per_term_docs},
        "hits": hits,
        "not_searched": not_searched,
        "unreadable": unreadable,
        "warning": ("A zero result is only meaningful alongside the coverage block above. "
                    "Items not searched were not examined and may contain the term."),
    }
    with open(a.out, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=1); f.write("\n")
    c = report["coverage"]
    print(f"searched={c['items_searched']}/{c['items_encountered']} "
          f"({c['coverage_percent']}%) unreadable={c['items_unreadable']} "
          f"not_searched={c['items_not_searched_unsupported_format']} -> {a.out}")
    if c["items_searched"] != c["items_encountered"]:
        print("WARNING: coverage is incomplete. Do not report a zero result without the coverage block.")

if __name__ == "__main__":
    main()