#!/usr/bin/env python3
"""
Build a catalog of Venus radar SigMF recordings.

Crawls one or more directories (recursively) for *.sigmf-meta files and writes
three tables to the output directory:

  files.csv          one row per received SigMF file, with per-file checks
  recordings.csv     one row per station per transmission; files that differ only in
                     channel (polarization, vrt:channel or '.chanN' in the name) are
                     combined into one row
  transmissions.csv  one row per transmit event, with the stations that recorded it

Each recording has a 'kind': 'radar' (description parsed as below), 'other' (a
description without ';', taken as the object name, e.g. '3C273'), or 'unknown'.
  waveforms.csv      one row per transmitted waveform file, with its length

The received files are linked to a transmission through core:description,
which holds  $TX_FILENAME;$TX_GAIN;$TXTIME;$OBJECT

Usage:
  python build_catalog.py sigmf/ [more_dirs ...] --tx-dir tx_waveforms/ -o catalog/

Subdirectories of --tx-dir named in TX_EXCLUDE (default: mfsk-v2) are skipped;
override with --tx-exclude.

Nothing in the data directories is read beyond the .sigmf-meta files and the
sizes of the .sigmf-data files; nothing is modified.
"""

import argparse
import json
import re
import sys
from pathlib import Path

import pandas as pd

# --------------------------------------------------------------------------
# Things to adapt
# --------------------------------------------------------------------------

# Datatype assumed when a meta file does not declare one.
DEFAULT_DATATYPE = "ci16_le"

# Directories (names or glob patterns, at any depth) under --tx-dir to leave out.
# mfsk-v2 holds the same file names as mfsk-v1, with messages that must stay secret.
TX_EXCLUDE = ["mfsk-v2"]

# Some waveforms come in variants that belong to one category. Each pattern is
# stripped from the end of the filename stem to get the category, e.g.
#   venus-mfsk4096-rs9-sp3-d4-269s-tx01  ->  venus-mfsk4096-rs9-sp3-d4-269s
VARIANT_PATTERNS = [re.compile(r"-tx\d+$")]

# Allowed mismatch between TXTIME and the recording's core:datetime.
# Recordings seen so far start about 1 s before TXTIME.
MAX_START_OFFSET_S = 2.0


def infer_station(meta_path: Path, global_: dict) -> tuple[str, str | None]:
    """Short station ID and full station name for one recording.

    The ID is the first underscore-separated part of the filename
    (dwingeloo_venus_2026_10_17_... -> 'dwingeloo'); the full name is
    core:author ('Dwingeloo Radio Telescope').
    """
    station = meta_path.name.split("_", 1)[0].lower()
    return station, global_.get("core:author")


# --------------------------------------------------------------------------
# SigMF helpers
# --------------------------------------------------------------------------

def bytes_per_sample(datatype: str, num_channels: int = 1) -> int:
    """Bytes per sample for a SigMF datatype string such as 'ci16_le'."""
    m = re.fullmatch(r"(c?)([fiu])(8|16|32|64)(_le|_be)?", datatype)
    if not m:
        raise ValueError(f"unknown SigMF datatype {datatype!r}")
    n = int(m.group(3)) // 8
    if m.group(1) == "c":
        n *= 2
    return n * num_channels


SKIPPED: list = []   # (path, reason) of .sigmf-meta files that couldn't be read


def try_read_sigmf(meta_path: Path) -> dict | None:
    """read_sigmf, or None (and an entry in SKIPPED) for an empty or broken meta file,
    e.g. one written while the disk was full. Such files are left out of the catalog."""
    try:
        if meta_path.stat().st_size == 0:
            raise ValueError("empty file")
        return read_sigmf(meta_path)
    except (ValueError, UnicodeDecodeError, AttributeError, TypeError) as e:
        # json.JSONDecodeError is a ValueError
        reason = str(e) if str(e) == "empty file" else f"not valid SigMF metadata ({e})"
        SKIPPED.append((meta_path, reason))
        return None


def read_sigmf(meta_path: Path) -> dict:
    """Header fields of one SigMF recording, plus its length from the data file size."""
    meta = json.loads(meta_path.read_text())
    g = meta.get("global", {})
    cap = (meta.get("captures") or [{}])[0]
    data_path = meta_path.with_suffix(".sigmf-data")

    datatype = g.get("core:datatype", DEFAULT_DATATYPE)
    num_channels = g.get("core:num_channels", 1)
    sample_rate = g.get("core:sample_rate")

    n_samples = duration = None
    if data_path.exists():
        n_samples = data_path.stat().st_size // bytes_per_sample(datatype, num_channels)
        if sample_rate:
            duration = n_samples / sample_rate

    lat = lon = None
    geo = g.get("core:geolocation")
    if isinstance(geo, dict) and geo.get("coordinates"):
        lon, lat = geo["coordinates"][:2]  # GeoJSON order is lon, lat

    # Keep every non-core field (e.g. vrt:rx_gain, vrt:time_source) as an extra
    # column, with ':' replaced by '_' so the names are valid identifiers.
    extra = {k.replace(":", "_"): v for k, v in {**g, **cap}.items()
             if not k.startswith("core:") and not isinstance(v, (dict, list))}

    return {
        "global": g,
        "extra": extra,
        "data_exists": data_path.exists(),
        "datatype": datatype,
        "sample_rate": sample_rate,
        "frequency": cap.get("core:frequency"),
        "start": pd.to_datetime(cap.get("core:datetime"), utc=True, errors="coerce"),
        "n_samples": n_samples,
        "duration_s": duration,
        "description": g.get("core:description", ""),
        "lat": lat,
        "lon": lon,
    }


# A polarization label at the end of the object field, e.g. 've002_ef_no0013 RCP'
# (Effelsberg puts its scan label and polarization there).
POL_SUFFIX = re.compile(r"^(.*?)\s+(RCP|LCP|RHCP|LHCP|R|L|X|Y|H|V)$", re.I)


def split_polarization(obj):
    """'ve002_ef_no0013 RCP' -> ('ve002_ef_no0013', 'RCP'); no label -> (obj, None)."""
    if isinstance(obj, str):
        m = POL_SUFFIX.match(obj.strip())
        if m:
            return m.group(1), m.group(2).upper()
    return obj, None


def parse_description(desc: str) -> dict | None:
    """Split '$TX_FILENAME;$TX_GAIN;$TXTIME;$OBJECT' into fields, or None."""
    parts = desc.split(";")
    if len(parts) != 4:
        return None
    tx_file, gain, txtime, obj = (p.strip() for p in parts)
    try:
        tx_time = pd.to_datetime(float(txtime), unit="s", utc=True)
        gain = float(gain)
    except ValueError:
        return None
    obj, pol = split_polarization(obj)
    return {"tx_file": tx_file, "tx_gain": gain, "tx_time": tx_time, "object": obj,
            "polarization": pol}


def waveform_stem(tx_file: str) -> str:
    name = Path(tx_file).name
    for ext in (".sigmf-meta", ".sigmf-data", ".sigmf"):
        name = name.removesuffix(ext)
    return name


def waveform_category(tx_file: str) -> str:
    stem = waveform_stem(tx_file)
    for pat in VARIANT_PATTERNS:
        stem = pat.sub("", stem)
    return stem


def tx_id_for(tx_time: pd.Timestamp) -> str:
    """Stable ID for a transmission; unique because transmissions don't overlap."""
    return tx_time.strftime("%Y%m%dT%H%M%SZ")


def as_utc(df: pd.DataFrame, cols: list[str]) -> pd.DataFrame:
    """Make time columns timezone-aware UTC, also when they are all NaT."""
    for c in cols:
        df[c] = pd.to_datetime(df[c], utc=True)
    return df


def iso_times(df: pd.DataFrame) -> pd.DataFrame:
    """Copy of df with all timezone-aware columns as fixed-format ISO 8601 strings,
    e.g. 2026-10-17T12:46:35.000145187Z (always 9 decimals, nanoseconds kept).

    pandas' default CSV output varies the precision per value, which newer pandas
    versions can't read back without format='ISO8601'.
    """
    out = df.copy()
    for col in out.columns:
        if isinstance(out[col].dtype, pd.DatetimeTZDtype):
            t = out[col].dt.tz_convert("UTC")
            frac = (t.dt.microsecond * 1000 + t.dt.nanosecond).map(
                lambda v: "" if pd.isna(v) else f"{int(v):09d}")
            out[col] = (t.dt.strftime("%Y-%m-%dT%H:%M:%S") + "." + frac + "Z").where(t.notna(), "")
    return out


def find_metas(dirs: list[Path], exclude=()) -> list[Path]:
    """All .sigmf-meta files under dirs, skipping any inside a directory whose name
    matches one of the `exclude` patterns."""
    from fnmatch import fnmatch
    found = set()
    for d in dirs:
        if not d.is_dir():
            print(f"warning: {d} is not a directory, skipped", file=sys.stderr)
            continue
        for p in d.rglob("*.sigmf-meta"):
            parts = p.relative_to(d).parts[:-1]
            if any(fnmatch(part, pat) for part in parts for pat in exclude):
                continue
            found.add(p.resolve())
    return sorted(found)


# --------------------------------------------------------------------------
# Catalog building
# --------------------------------------------------------------------------

def build_waveforms(tx_dirs: list[Path], exclude=()) -> pd.DataFrame:
    rows = []
    for p in find_metas(tx_dirs, exclude):
        s = try_read_sigmf(p)
        if s is None:
            continue
        rows.append({
            "tx_file": p.name,
            "category": waveform_category(p.name),
            "variant": waveform_stem(p.name),
            "sample_rate": s["sample_rate"],
            "frequency": s["frequency"],
            "n_samples": s["n_samples"],
            "tx_duration_s": s["duration_s"],
            "path": str(p),
        })
    cols = ["tx_file", "category", "variant", "sample_rate", "frequency",
            "n_samples", "tx_duration_s", "path"]
    wf = pd.DataFrame(rows, columns=cols).sort_values(["category", "variant", "path"])

    # The received metadata only names the file, so a name must be unique.
    dup = wf[wf.duplicated("tx_file", keep=False)]
    for name, g in dup.groupby("tx_file"):
        lengths = ", ".join(f"{d:.3f} s" for d in g["tx_duration_s"].unique())
        print(f"warning: TX waveform {name} found {len(g)} times ({lengths}); using "
              f"{g['path'].iloc[0]}. Exclude the others with --tx-exclude.", file=sys.stderr)
    return wf.drop_duplicates("tx_file", keep="first").reset_index(drop=True)


def infer_channel(meta_path: Path, global_: dict) -> int:
    """Channel (polarization) number: vrt:channel, else '.chanN' in the filename, else 0."""
    if global_.get("vrt:channel") is not None:
        return int(global_["vrt:channel"])
    m = re.search(r"\.chan(\d+)$", meta_path.name.removesuffix(".sigmf-meta"))
    return int(m.group(1)) if m else 0


def build_files(rx_dirs: list[Path], waveforms: pd.DataFrame) -> pd.DataFrame:
    """One row per received SigMF file, with per-file checks."""
    wf = waveforms.set_index("tx_file")
    names_seen: dict[str, set] = {}
    rows = []
    for p in find_metas(rx_dirs):
        s = try_read_sigmf(p)
        if s is None:
            continue
        d = parse_description(s["description"])
        issues = []
        if not s["data_exists"]:
            issues.append(f"data file not found: {p.with_suffix('.sigmf-data').name}")
        # kind: 'radar' = echo recording of a transmission,
        #       'other' = description without ';', taken as the observed object (e.g. '3C273'),
        #       'unknown' = description missing or malformed.
        kind = "radar"
        if d is None:
            desc = s["description"].strip()
            obj = None
            if desc and ";" not in desc:
                kind, obj = "other", desc
            else:
                kind = "unknown"
                if desc:
                    issues.append(f"unparsable core:description {desc!r} "
                                  "(expected TX_FILENAME;TX_GAIN;TXTIME;OBJECT)")
                else:
                    issues.append("no core:description")
            obj, pol = split_polarization(obj)
            d = {"tx_file": None, "tx_gain": None, "tx_time": pd.NaT, "object": obj,
                 "polarization": pol}

        tx_duration = None
        if d["tx_file"] and not wf.empty:
            if d["tx_file"] in wf.index:
                tx_duration = wf.at[d["tx_file"], "tx_duration_s"]
            else:
                issues.append(f"tx waveform not found in --tx-dir: {d['tx_file']}")

        start_offset = (s["start"] - d["tx_time"]).total_seconds() \
            if pd.notna(s["start"]) and pd.notna(d["tx_time"]) else None
        if start_offset is not None and abs(start_offset) > MAX_START_OFFSET_S:
            issues.append(f"starts {start_offset:+.1f}s from TXTIME")

        listen = None
        if s["duration_s"] is not None and tx_duration is not None:
            listen = s["duration_s"] - tx_duration
            if listen < 0:
                issues.append(f"recording ({s['duration_s']:.1f}s) shorter than "
                              f"transmission ({tx_duration:.1f}s)")

        station, station_name = infer_station(p, s["global"])
        names_seen.setdefault(station, set()).add(station_name)
        if not station_name:
            issues.append("no core:author")
        elif station not in station_name.lower():
            issues.append(f"filename station {station!r} vs core:author {station_name!r}")

        day_from = d["tx_time"] if pd.notna(d["tx_time"]) else s["start"]
        rows.append({
            "kind": kind,
            "tx_id": tx_id_for(d["tx_time"]) if pd.notna(d["tx_time"]) else None,
            "station": station,
            "station_name": station_name,
            "channel": infer_channel(p, s["global"]),
            "polarization": d["polarization"],
            "date": day_from.date() if pd.notna(day_from) else None,
            "tx_time": d["tx_time"],
            "rec_start": s["start"],
            "start_offset_s": start_offset,
            "duration_s": s["duration_s"],
            "tx_duration_s": tx_duration,
            # time recorded after the transmission ends; must exceed the round-trip light time
            "listen_s": listen,
            "tx_file": d["tx_file"],
            "category": waveform_category(d["tx_file"]) if d["tx_file"] else None,
            "tx_gain": d["tx_gain"],
            "object": d["object"],
            "sample_rate": s["sample_rate"],
            "frequency": s["frequency"],
            "datatype": s["datatype"],
            "n_samples": s["n_samples"],
            "lat": s["lat"],
            "lon": s["lon"],
            "path": str(p),
            "issues": " | ".join(issues),
            **s["extra"],
        })
    df = pd.DataFrame(rows)
    if df.empty:
        return df
    as_utc(df, ["tx_time", "rec_start"])
    df["n_samples"] = df["n_samples"].astype("Int64")  # integer, also with missing data files

    for station, names in names_seen.items():
        if len(names) > 1:
            print(f"warning: station {station!r} has several core:author values: {names}",
                  file=sys.stderr)

    return df.sort_values(["rec_start", "station", "channel"]).reset_index(drop=True)


# Columns that must be identical for all channels of one recording.
MUST_AGREE = ["rec_start", "sample_rate", "frequency", "datatype", "n_samples",
              "tx_file", "tx_gain", "object"]


def build_recordings(files: pd.DataFrame) -> pd.DataFrame:
    """One row per station per transmission; the channels (polarizations) are combined.

    'channels' (comma-separated) and 'paths' (separated by ' | ') list the channel
    numbers and meta files in the same order.
    """
    files = files.copy()
    # Group by transmission and station. Files whose transmission is unknown (e.g.
    # non-radar observations) are grouped by filename with '.chanN' removed.
    files["_key"] = files["tx_id"].fillna("?") + "/" + files["station"]
    no_tx = files["tx_id"].isna()
    files.loc[no_tx, "_key"] = files.loc[no_tx, "path"].map(
        lambda p: re.sub(r"\.chan\d+$", "", p.removesuffix(".sigmf-meta")))

    rows = []
    for _, g in files.groupby("_key", sort=False):
        g = g.sort_values("channel")
        multi = len(g) > 1
        issues = []
        # Per-file issues: reported once if all files have it, else labelled by channel.
        per_file = [(f["channel"], list(filter(None, f["issues"].split(" | "))))
                    for _, f in g.iterrows()]
        for ch, file_issues in per_file:
            for issue in file_issues:
                in_all = all(issue in other for _, other in per_file)
                if not (multi and in_all):
                    issues.append(f"chan{ch}: {issue}" if multi else issue)
                elif issue not in issues:
                    issues.append(issue)
        dup = g["channel"][g["channel"].duplicated()].unique()
        for ch in dup:
            names = ", ".join(Path(p).name for p in g.loc[g["channel"] == ch, "path"])
            issues.append(f"channel {ch} appears in several files: {names}")
        for col in MUST_AGREE:
            if g[col].nunique(dropna=False) > 1:
                vals = ", ".join(f"chan{c}={v}" for c, v in zip(g["channel"], g[col]))
                issues.append(f"channels differ in {col}: {vals}")

        row = {}
        for col in g.columns.drop(["_key", "path", "issues", "channel"]):
            vals = g[col]
            if vals.nunique(dropna=False) <= 1 or col in MUST_AGREE:
                row[col] = vals.iloc[0]
            else:  # e.g. vrt_stream_id, vrt_frac_frequency
                row[col] = ",".join(str(v) for v in vals)
        # Use the common (shortest) length if the channels differ.
        row["n_samples"] = g["n_samples"].min()
        row["duration_s"] = g["duration_s"].min()
        if pd.notna(row["duration_s"]) and pd.notna(row["tx_duration_s"]):
            row["listen_s"] = row["duration_s"] - row["tx_duration_s"]
        row["n_channels"] = g["channel"].nunique()
        row["channels"] = ",".join(str(c) for c in g["channel"])
        row["paths"] = " | ".join(g["path"])
        row["issues"] = " | ".join(issues)
        rows.append(row)

    rec = pd.DataFrame(rows)
    as_utc(rec, ["tx_time", "rec_start"])
    rec["n_samples"] = rec["n_samples"].astype("Int64")

    # A station that usually records N channels but has fewer here is missing a polarization.
    usual = rec.groupby("station")["n_channels"].transform("max")
    short = rec["n_channels"] < usual
    rec.loc[short, "issues"] = [
        " | ".join(filter(None, [iss, f"only channel(s) {ch} present, station usually has {u}"]))
        for iss, ch, u in zip(rec.loc[short, "issues"], rec.loc[short, "channels"], usual[short])]
    # Put the channel columns right after the station columns.
    cols = list(rec.columns)
    front = ["n_channels", "channels", "polarization"]
    for c in front:
        cols.remove(c)
    i = cols.index("station_name") + 1
    rec = rec[cols[:i] + front + cols[i:]]
    return rec.sort_values(["rec_start", "station"]).reset_index(drop=True)


def build_transmissions(rec: pd.DataFrame) -> pd.DataFrame:
    rec = rec[rec["tx_id"].notna()].copy()
    rec["rec_end"] = rec["rec_start"] + pd.to_timedelta(rec["duration_s"], unit="s")

    def summarize(g: pd.DataFrame) -> dict:
        issues = []
        # tx_file and tx_gain come from the transmitter and must agree; the object field
        # is each station's own label (e.g. Effelsberg's scan name), so it may differ.
        for col in ("tx_file", "tx_gain"):
            if g[col].nunique(dropna=False) > 1:
                per_station = ", ".join(f"{st}={v}" for st, v in zip(g["station"], g[col]))
                issues.append(f"stations disagree on {col}: {per_station}")
        first = g.iloc[0]
        tx_end = first["tx_time"] + pd.to_timedelta(first["tx_duration_s"], unit="s") \
            if pd.notna(first["tx_duration_s"]) else pd.NaT
        return {
            "date": first["date"],
            "tx_time": first["tx_time"],
            "tx_end": tx_end,
            "rec_end": g["rec_end"].max(),
            "tx_duration_s": first["tx_duration_s"],
            "category": first["category"],
            "tx_file": first["tx_file"],
            "tx_gain": first["tx_gain"],
            "object": g["object"].mode().iloc[0] if g["object"].notna().any() else None,
            "n_stations": g["station"].nunique(),
            "stations": ",".join(sorted(g["station"].unique())),
            "issues": " | ".join(issues),
        }

    # A plain loop rather than groupby().apply(), which changed behaviour across pandas versions.
    tx = pd.DataFrame([{"tx_id": k, **summarize(g)} for k, g in rec.groupby("tx_id")])
    as_utc(tx, ["tx_time", "tx_end", "rec_end"])
    tx = tx.sort_values("tx_time").reset_index(drop=True)

    # Transmissions should not overlap.
    overlap = tx["tx_end"] > tx["tx_time"].shift(-1)
    tx.loc[overlap, "issues"] = tx.loc[overlap, "issues"].map(
        lambda s: " | ".join(filter(None, [s, "overlaps next transmission"])))
    return tx


def coverage_matrix(rec: pd.DataFrame) -> pd.DataFrame:
    """tx_id x station table: number of channels recorded, '.' if none."""
    m = rec.pivot_table(index="tx_id", columns="station", values="n_channels", aggfunc="sum")
    return m.fillna(0).astype(int).astype(str).replace("0", ".")


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("rx_dirs", nargs="+", type=Path,
                    help="directories to crawl for received SigMF recordings")
    ap.add_argument("--tx-exclude", action="append", default=None, metavar="PATTERN",
                    help="directory name (glob) under --tx-dir to skip, repeatable; "
                         f"default: {' '.join(TX_EXCLUDE)}. Use --tx-exclude '' to skip nothing.")
    ap.add_argument("--tx-dir", action="append", type=Path, default=[],
                    help="directory with transmitted waveform SigMFs (repeatable)")
    ap.add_argument("-o", "--out", type=Path, default=Path("catalog"),
                    help="output directory (default: catalog/)")
    args = ap.parse_args(argv)

    exclude = [e for e in (args.tx_exclude if args.tx_exclude is not None else TX_EXCLUDE) if e]
    waveforms = build_waveforms(args.tx_dir, exclude)
    if waveforms.empty:
        print("warning: no transmitted waveforms found (use --tx-dir); "
              "TX durations and listen_s will be empty", file=sys.stderr)
    files = build_files(args.rx_dirs, waveforms)
    if files.empty:
        sys.exit("no .sigmf-meta files found")
    recordings = build_recordings(files)
    transmissions = build_transmissions(recordings)

    args.out.mkdir(parents=True, exist_ok=True)
    waveforms.to_csv(args.out / "waveforms.csv", index=False)
    iso_times(files).to_csv(args.out / "files.csv", index=False)
    iso_times(recordings).to_csv(args.out / "recordings.csv", index=False)
    iso_times(transmissions).to_csv(args.out / "transmissions.csv", index=False)

    # Short report
    pd.set_option("display.width", 160)
    pd.set_option("display.max_columns", 20)
    kinds = ", ".join(f"{n} {k}" for k, n in recordings["kind"].value_counts().items())
    print(f"{len(files)} files, {len(recordings)} recordings ({kinds}), "
          f"{len(transmissions)} transmissions, "
          f"{recordings['station'].nunique()} stations, "
          f"{recordings['category'].nunique()} waveform categories, "
          f"{recordings['date'].nunique()} days\n")
    other = recordings[recordings["kind"] == "other"]
    if len(other):
        print("Other observations:")
        for _, r in other.iterrows():
            start = f"{r['rec_start']:%Y-%m-%d %H:%M:%S}" if pd.notna(r["rec_start"]) else "?"
            dur = f"{r['duration_s']:7.1f}s" if pd.notna(r["duration_s"]) else "      ?s"
            print(f"  {start}  {r['station']:<10} {r['object']:<12} {dur}  {r['n_channels']} ch")
        print()
    print("Transmissions per category:")
    print(transmissions["category"].value_counts().to_string(), "\n")
    print("Coverage (number of channels recorded):")
    print(coverage_matrix(recordings).to_string(), "\n")
    if SKIPPED:
        print(f"Skipped, not in the catalog ({len(SKIPPED)} unreadable .sigmf-meta):")
        for path, reason in SKIPPED:
            print(f"  {path.name}: {reason}")
        print()
    problems = recordings[recordings["issues"] != ""]
    tx_problems = transmissions[transmissions["issues"] != ""]
    if len(problems) or len(tx_problems):
        print("Issues:")
        for _, r in problems.iterrows():
            names = [Path(p).name for p in r["paths"].split(" | ")]
            for name in names[:-1]:
                print(f"  {name}")
            print(f"  {names[-1]}:")
            for issue in r["issues"].split(" | "):
                print(f"      {issue}")
        for _, r in tx_problems.iterrows():
            print(f"  transmission {r['tx_id']}:")
            for issue in r["issues"].split(" | "):
                print(f"      {issue}")
    else:
        print("No issues found.")
    print(f"\nWritten to {args.out}/")


if __name__ == "__main__":
    main()
