"""
Venus-bounce M-ary FSK: shared code for the tx, channel-simulation and rx notebooks.

Signal chain
    text (A-Z, 0-9, space) -> base-37 integer -> + CRC (16 or 24 bit) -> k symbols of log2(M) bits
    -> RS(n, k) over GF(M) -> n code symbols -> dwell schedule (dwell j carries symbol j mod n)
    -> one tone per dwell, tones centered on DC, every tone on an exact FFT bin of the frame

Timing
    frame    = round(sample_rate / bin_hz) samples, the detection FFT (bin = echo Doppler spread)
    dwell    = dwell_frames frames on one tone
    tx       = n_dwells dwells; the requested tx_length_s is shortened to an integer number of dwells

Every tone completes an integer number of cycles per frame, so the phase is continuous across
tone changes without any phase tracking.
"""

import datetime
import glob
import hashlib
import itertools
import json
import math
import os
import re
from dataclasses import asdict, dataclass, fields

import galois
import numpy as np
from sigmf import SigMFFile, sigmffile

ALPHABET = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"  # space = 0, messages are right-padded with spaces

# CRC width -> (polynomial, init), computed bitwise over the message bits (MSB first)
CRC_SPECS = {
    16: (0x1021, 0xFFFF),           # CRC-16/CCITT-FALSE
    24: (0x864CFB, 0xB704CE),       # CRC-24/OPENPGP
}


# ---------------------------------------------------------------------------------------------
# Parameters
# ---------------------------------------------------------------------------------------------

@dataclass
class MFSKParams:
    sample_rate: float = 250e3          # Hz
    center_freq: float = 1299.650e6     # Hz, SDR tuning frequency; the tone block is centered on DC
    tx_length_s: float = 300.0          # requested transmission length, shortened to whole dwells
    bin_hz: float = 1.5                 # detection bin width = echo Doppler spread; frame = 1 / bin_hz
    M: int = 4096                       # number of tones, power of 2 (log2(M) bits per code symbol)
    tone_spacing_bins: int = 5          # tone spacing in detection bins
    dwell_frames: int = 4               # frames per dwell (one tone per dwell)
    n_chars: int = 11                   # message length in characters, padded with spaces
    crc_bits: int = 24                  # CRC width, a key of CRC_SPECS
    rs_n: int = 12                      # RS code length in symbols (rs_k follows from the message)
    scramble: bool = False              # tone scrambling: tone = symbol XOR PN(seed, tx index, dwell)
    scramble_seed: int = 1              # seed of the scrambling sequence (one set of tx files per seed)
    max_bandwidth_hz: float = 150e3     # usable bandwidth, for validation only

    def __post_init__(self):
        self.validate()

    # --- derived timing ---
    @property
    def frame_samples(self):
        return int(round(self.sample_rate / self.bin_hz))

    @property
    def frame_s(self):
        return self.frame_samples / self.sample_rate

    @property
    def bin_actual_hz(self):
        return self.sample_rate / self.frame_samples

    @property
    def dwell_samples(self):
        return self.frame_samples * self.dwell_frames

    @property
    def n_dwells(self):
        return int(self.tx_length_s * self.sample_rate // self.dwell_samples)

    @property
    def n_frames(self):
        return self.n_dwells * self.dwell_frames

    @property
    def tx_samples(self):
        return self.n_dwells * self.dwell_samples

    @property
    def tx_actual_s(self):
        return self.tx_samples / self.sample_rate

    # --- derived tones ---
    @property
    def bits_per_symbol(self):
        return int(round(math.log2(self.M)))

    @property
    def tone_spacing_hz(self):
        return self.tone_spacing_bins * self.bin_actual_hz

    @property
    def bandwidth_hz(self):
        return self.M * self.tone_spacing_hz

    @property
    def tone_bins(self):
        """Signed FFT bin of every tone, tone M/2 at DC."""
        return (np.arange(self.M, dtype=np.int64) - self.M // 2) * self.tone_spacing_bins

    @property
    def tone_freqs_hz(self):
        return self.tone_bins * self.bin_actual_hz

    # --- derived coding ---
    @property
    def msg_bits(self):
        return (len(ALPHABET) ** self.n_chars - 1).bit_length()

    @property
    def rs_k(self):
        return -(-(self.msg_bits + self.crc_bits) // self.bits_per_symbol)

    @property
    def pad_bits(self):
        return self.rs_k * self.bits_per_symbol - self.msg_bits - self.crc_bits

    def dwell_symbol(self, dwell):
        """Code symbol index carried by a dwell (plain cyclic schedule)."""
        return np.asarray(dwell) % self.rs_n

    @property
    def dwells_per_symbol(self):
        return np.bincount(self.dwell_symbol(np.arange(self.n_dwells)), minlength=self.rs_n)

    def validate(self):
        if self.crc_bits not in CRC_SPECS:
            raise ValueError("crc_bits must be one of %s" % sorted(CRC_SPECS))
        m = math.log2(self.M)
        if m != int(m):
            raise ValueError("M must be a power of 2")
        if self.bandwidth_hz > self.max_bandwidth_hz:
            raise ValueError("tone block %.1f kHz exceeds max_bandwidth_hz %.1f kHz"
                             % (self.bandwidth_hz / 1e3, self.max_bandwidth_hz / 1e3))
        if self.bandwidth_hz >= self.sample_rate:
            raise ValueError("tone block wider than the sample rate")
        if not self.rs_k < self.rs_n <= self.M - 1:
            raise ValueError("need rs_k (%d) < rs_n (%d) <= M-1" % (self.rs_k, self.rs_n))
        if self.n_dwells < self.rs_n:
            raise ValueError("only %d dwells for %d code symbols; lengthen tx or shorten dwells"
                             % (self.n_dwells, self.rs_n))

    def summary(self):
        dps = self.dwells_per_symbol
        lines = [
            "frame        %d samples = %.4f s, bin %.5f Hz" % (self.frame_samples, self.frame_s, self.bin_actual_hz),
            "dwell        %d frames = %.4f s" % (self.dwell_frames, self.dwell_samples / self.sample_rate),
            "tx length    %d dwells = %d frames = %.3f s (requested %.1f s)"
            % (self.n_dwells, self.n_frames, self.tx_actual_s, self.tx_length_s),
            "tones        M=%d, spacing %d bins = %.4f Hz, block %.3f kHz centered on DC"
            % (self.M, self.tone_spacing_bins, self.tone_spacing_hz, self.bandwidth_hz / 1e3),
            "message      %d chars -> %d bits + CRC-%d + %d pad = %d bits = %d symbols of %d bits"
            % (self.n_chars, self.msg_bits, self.crc_bits, self.pad_bits, self.rs_k * self.bits_per_symbol,
               self.rs_k, self.bits_per_symbol),
            "code         RS(%d, %d) over GF(%d), rate %.3f"
            % (self.rs_n, self.rs_k, self.M, self.rs_k / self.rs_n),
            "schedule     each code symbol in %d..%d dwells (%.1f..%.1f s) per transmission"
            % (dps.min(), dps.max(), dps.min() * self.dwell_samples / self.sample_rate,
               dps.max() * self.dwell_samples / self.sample_rate),
            "scrambling   " + ("on, seed %d (tone = symbol XOR PN per tx index and dwell)" % self.scramble_seed
                               if self.scramble else "off"),
        ]
        return "\n".join(lines)

    # --- SigMF round trip ---
    def to_sigmf(self):
        return {"mfsk:" + k: v for k, v in asdict(self).items()}

    @classmethod
    def from_sigmf(cls, meta_path, **overrides):
        """Rebuild the parameters from the mfsk:* fields of a tx SigMF file (only the .sigmf-meta is needed)."""
        g = read_global(meta_path)
        kw = {f.name: g["mfsk:" + f.name] for f in fields(cls) if "mfsk:" + f.name in g}
        kw.update(overrides)
        return cls(**kw)


# ---------------------------------------------------------------------------------------------
# Message <-> integer <-> symbols
# ---------------------------------------------------------------------------------------------

def crc(value, nbits, width=24):
    poly, reg = CRC_SPECS[width]
    mask = (1 << width) - 1
    for i in range(nbits - 1, -1, -1):
        top = (reg >> (width - 1)) & 1
        reg = (reg << 1) & mask
        if top ^ ((value >> i) & 1):
            reg ^= poly
    return reg


def crc24(value, nbits):
    return crc(value, nbits, 24)


def text_to_value(text, n_chars):
    text = text.upper()
    if len(text) > n_chars:
        raise ValueError("message longer than %d characters" % n_chars)
    text = text.ljust(n_chars)
    value = 0
    for ch in text:
        i = ALPHABET.find(ch)
        if i < 0:
            raise ValueError("character %r not in alphabet A-Z 0-9 space" % ch)
        value = value * len(ALPHABET) + i
    return value


def value_to_text(value, n_chars):
    chars = []
    for _ in range(n_chars):
        value, i = divmod(value, len(ALPHABET))
        chars.append(ALPHABET[i])
    return "".join(reversed(chars))


def text_to_symbols(text, p):
    """Message -> rs_k information symbols (pad bits | message bits | CRC, MSB first)."""
    value = text_to_value(text, p.n_chars)
    word = (value << p.crc_bits) | crc(value, p.msg_bits, p.crc_bits)
    m = p.bits_per_symbol
    return np.array([(word >> (m * (p.rs_k - 1 - j))) & (p.M - 1) for j in range(p.rs_k)], dtype=np.int64)


def symbols_to_text(symbols, p):
    """Information symbols -> (text, None) if all checks pass, else (None, reason)."""
    word = 0
    for s in symbols:
        word = (word << p.bits_per_symbol) | int(s)
    check = word & ((1 << p.crc_bits) - 1)
    value = word >> p.crc_bits
    if value >> p.msg_bits:
        return None, "pad bits"
    if value >= len(ALPHABET) ** p.n_chars:
        return None, "range"
    if crc(value, p.msg_bits, p.crc_bits) != check:
        return None, "crc"
    return value_to_text(value, p.n_chars), None


# ---------------------------------------------------------------------------------------------
# Reed-Solomon code and decoder
# ---------------------------------------------------------------------------------------------

@dataclass
class DecodeResult:
    ok: bool
    text: str = None
    codeword: np.ndarray = None
    score: float = None         # sum of normalized energies at the codeword tones
    zscore: float = None        # (score - noise mean) / noise std of that sum
    n_agree: int = None         # code symbols where the codeword tone is the strongest tone
    best_score: float = None    # best score of any candidate, valid or not
    n_candidates: int = 0
    phase: int = None           # decoder pass that found it: 1 = OSD on k positions, 2 = k-1 positions + free symbol
    method: str = None          # position set of that pass and list ranks of the codeword tones there

    def __str__(self):
        if not self.ok:
            return "no decode (%d candidates, best score %.1f)" % (self.n_candidates, self.best_score or 0)
        return ("'%s'  z = %.1f, %d/%d symbols agree with hard decisions, %d candidates"
                % (self.text, self.zscore, self.n_agree, len(self.codeword), self.n_candidates))


class Codec:
    """RS(n, k) over GF(M), built as a shortened RS(M-1, M-1-(n-k)) code (systematic, message first)."""

    def __init__(self, p):
        self.p = p
        self.GF = galois.GF(p.M)
        self.rs = galois.ReedSolomon(p.M - 1, p.M - 1 - (p.rs_n - p.rs_k), field=self.GF)
        self.G = self.rs.encode(self.GF.Identity(p.rs_k))   # k x n generator matrix

    def encode_symbols(self, info):
        return np.asarray(self.rs.encode(self.GF(info)), dtype=np.int64)

    def encode(self, text):
        return self.encode_symbols(text_to_symbols(text, self.p))

    def check(self, codeword):
        return symbols_to_text(codeword[:self.p.rs_k], self.p)

    def decode(self, E, L, V=None, n_bases=128, list_depth=3, n_check=16, rank_scale=1.0, seed=0,
               n_free_sets=16, free_depth=5, min_z=None):
        """
        Soft decode the accumulated energy matrix E (n x M, noise-normalized, noise mean L[i] and
        variance V[i] per row; V defaults to L, independent frames).

        Ordered-statistics decoding: pick k positions (the k most reliable first, then random sets
        weighted towards reliable positions), try every combination of their list_depth strongest
        tones, re-encode (MDS: any k symbols fix the codeword) and score sum_i E[i, c_i], which is the
        maximum-likelihood metric for square-law detection at low SNR.
        If that finds nothing, a second pass needs only k-1 positions in the lists: fix k-1 positions
        (the n_free_sets most reliable sets) to their free_depth strongest tones and enumerate the
        remaining information symbol over all M values (n_free_sets = 0 turns this pass off).
        The highest-scoring candidate that passes pad, range and CRC checks wins. Only the n_check
        best candidates of each pass are checked, which keeps the false-decode rate low; min_z also
        rejects a decode whose z-score is below it.
        """
        p, GF = self.p, self.GF
        n, k = p.rs_n, p.rs_k
        E = np.asarray(E, dtype=np.float64)
        L = np.asarray(L, dtype=np.float64)
        V = L if V is None else np.asarray(V, dtype=np.float64)
        rows = np.arange(n)

        order = np.argsort(-E, axis=1)[:, :max(list_depth, free_depth, 2)]
        rel = (E[rows, order[:, 0]] - E[rows, order[:, 1]]) / np.sqrt(V)
        by_rel = np.argsort(-rel)
        hard = order[:, 0]

        # bases after the first: sampled with a weight that falls off with reliability rank
        w = np.empty(n)
        w[by_rel] = np.exp(-np.arange(n) / (rank_scale * k))
        w /= w.sum()
        rng = np.random.default_rng(seed)
        combos = np.array(list(itertools.product(range(list_depth), repeat=k)), dtype=np.int64)

        def ranks(cw, S):
            """1-based rank of the codeword tone among the strongest tones at positions S ('>' beyond the list)."""
            return ",".join(str(int(np.flatnonzero(order[s] == cw[s])[0]) + 1) if cw[s] in order[s] else ">"
                            for s in S)

        best_sc, best_cw, origin, tried = [], [], [], set()
        for b in range(n_bases):
            S = np.sort(by_rel[:k] if b == 0 else rng.choice(n, size=k, replace=False, p=w))
            if tuple(S) in tried:
                continue
            tried.add(tuple(S))
            P = np.linalg.inv(self.G[:, S]) @ self.G       # k x n: basis symbols -> codeword
            Vb = GF(order[S[None, :], combos])              # (t^k, k) tone values at the basis positions
            C = np.asarray(Vb @ P, dtype=np.int64)           # (t^k, n) candidate codewords
            sc = E[rows[None, :], C].sum(axis=1)
            keep = np.argsort(-sc)[:n_check]
            best_sc.append(sc[keep])
            best_cw.append(C[keep])
            origin += [(b, S)] * len(keep)
        n_cand = len(tried) * len(combos)
        r, i = self._check_best(np.concatenate(best_sc), np.concatenate(best_cw), n_check, min_z, L, V, hard, n_cand)
        if r.ok:
            b, S = origin[i]
            r.phase = 1
            r.method = ("OSD basis %d of %d (%s), positions %s, list ranks %s"
                        % (b, n_bases, "most reliable" if b == 0 else "random", ",".join(map(str, S)),
                           ranks(r.codeword, S)))
        if r.ok or n_free_sets <= 0:
            return r

        # second pass: the codewords through k-1 fixed symbols form a line c = base + u * P_f over GF(M)
        # (addition = XOR), u the remaining information symbol, so every combination of the k-1 lists
        # is scored for all M values of u at once (float32: ~50 MB per position set at depth 5)
        sets = sorted(itertools.combinations(range(n), k - 1), key=lambda S: -rel[list(S)].sum())[:n_free_sets]
        combos = np.array(list(itertools.product(range(free_depth), repeat=k - 1)), dtype=np.int64)
        u = GF(np.arange(p.M))
        Ef = E.astype(np.float32)
        osd_best, best_cw, origin = r.best_score, [], []
        for j_set, S in enumerate(sets):
            S = np.array(S)
            f, *R = np.setdiff1d(rows, S)                                # free position, the others
            P = np.linalg.inv(self.G[:, np.r_[S, f]]) @ self.G           # rows: S, then the free symbol
            base = np.asarray(GF(order[S[None, :], combos]) @ P[:-1], dtype=np.int64)   # (t^(k-1), n), 0 at f
            line = np.asarray(u[:, None] * P[-1][None, :], dtype=np.int64)              # (M, n), u at f
            sc = Ef[S, base[:, S]].sum(axis=1)[:, None] + Ef[f][None, :]
            for j in R:
                sc += Ef[j][base[:, j, None] ^ line[None, :, j]]
            ci, ui = np.unravel_index(np.argpartition(-sc, n_check, axis=None)[:n_check], sc.shape)
            best_cw.append(base[ci] ^ line[ui])
            origin += [(j_set, S, f)] * len(ci)
        n_cand += len(sets) * len(combos) * p.M
        C = np.concatenate(best_cw)
        sc = E[rows[None, :], C].sum(axis=1)                # rescored in float64
        r, i = self._check_best(sc, C, n_check, min_z, L, V, hard, n_cand)
        r.best_score = max(r.best_score, osd_best)
        if r.ok:
            j_set, S, f = origin[i]
            r.phase = 2
            r.method = ("k-1 positions + free symbol, set %d of %d by reliability, positions %s, list ranks %s, "
                        "free position %d (tone rank %d of %d)"
                        % (j_set, len(sets), ",".join(map(str, S)), ranks(r.codeword, S), f,
                           int((E[f] > E[f, r.codeword[f]]).sum()) + 1, p.M))
        return r

    def _check_best(self, sc, C, n_check, min_z, L, V, hard, n_cand):
        """
        Check the n_check best distinct candidates; the best one that passes wins. Returns (result, index of the
        winning candidate or None); among equal candidates the first one counts (stable sort).
        """
        z = (sc - L.sum()) / np.sqrt(V.sum())
        seen = set()
        for i in np.argsort(-sc, kind="stable"):
            key = C[i].tobytes()
            if key in seen:
                continue
            seen.add(key)
            if min_z is not None and z[i] < min_z:
                break
            text, _ = self.check(C[i])
            if text is not None:
                return DecodeResult(
                    ok=True, text=text, codeword=C[i], score=float(sc[i]), zscore=float(z[i]),
                    n_agree=int((C[i] == hard).sum()), best_score=float(sc.max()), n_candidates=n_cand), i
            if len(seen) >= n_check:
                break
        return DecodeResult(ok=False, best_score=float(sc.max()), n_candidates=n_cand), None


# ---------------------------------------------------------------------------------------------
# SigMF helpers
# ---------------------------------------------------------------------------------------------

def read_global(meta):
    """Global section of a .sigmf-meta file (plain JSON read: no data file or checksum needed)."""
    with open(meta_path(meta)) as f:
        return json.load(f)["global"]


def utc_now_iso():
    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")


def parse_iso(ts):
    """ISO-8601 UTC string -> np.datetime64[ns] (nanosecond precision, 'Z' or '+00:00' accepted)."""
    ts = ts.strip().replace("+00:00", "").rstrip("Z")
    return np.datetime64(ts, "ns")


def format_iso(t):
    return str(np.datetime64(t, "ns")) + "Z"


def meta_path(base):
    return base if base.endswith(".sigmf-meta") else base + ".sigmf-meta"


def data_path(base):
    return (base[:-len(".sigmf-meta")] if base.endswith(".sigmf-meta") else base) + ".sigmf-data"


def write_ci16(f, x, scale):
    iq = np.empty(2 * len(x), dtype=np.int16)
    iq[0::2] = np.clip(np.round(x.real * scale), -32767, 32767)
    iq[1::2] = np.clip(np.round(x.imag * scale), -32767, 32767)
    iq.tofile(f)
    return np.abs(iq).max()


def write_meta(base, global_info, center_freq, datetime_iso, annotations=()):
    meta = SigMFFile(data_file=data_path(base), global_info={
        SigMFFile.DATATYPE_KEY: "ci16_le", **global_info})
    meta.add_capture(0, metadata={SigMFFile.FREQUENCY_KEY: center_freq, SigMFFile.DATETIME_KEY: datetime_iso})
    for start, length, md in annotations:
        meta.add_annotation(start, length, metadata=md)
    meta.tofile(meta_path(base))
    return meta_path(base)


def open_recording(path, pol_phase_deg=0.0, equalize_power=False):
    """
    A Recording, or a PolPair when path is a (chan0, chan1) pair of files. An already open recording
    (any object with read(), e.g. sigmf_dedoppler.DopplerCorrected) is returned as it is.
    """
    if hasattr(path, "read"):
        return path
    if isinstance(path, (list, tuple)):
        return PolPair(path[0], path[1], pol_phase_deg, equalize_power)
    return Recording(path)


def recording_label(path):
    if hasattr(path, "read"):
        return path.label
    if isinstance(path, (list, tuple)):
        return CHAN_RE.sub("chan0+1", os.path.basename(meta_path(path[0])))
    return os.path.basename(meta_path(path))


class Recording:
    """A SigMF recording, read as complex64 (full scale = 1). Reads outside the file return zeros."""

    def __init__(self, path):
        self.meta = sigmffile.fromfile(meta_path(path))
        self.sample_rate = self.meta.get_global_field(SigMFFile.SAMPLE_RATE_KEY)
        cap = self.meta.get_captures()[0]
        self.start_time = parse_iso(cap[SigMFFile.DATETIME_KEY])
        self.sample_start = cap.get(SigMFFile.START_INDEX_KEY, 0)
        self.center_freq = cap.get(SigMFFile.FREQUENCY_KEY)
        self.n_samples = self.meta.sample_count

    def sample_index(self, t_iso):
        dt_ns = (parse_iso(t_iso) - self.start_time).astype("timedelta64[ns]").astype(np.int64)
        return int(round(dt_ns * 1e-9 * self.sample_rate)) + self.sample_start

    def contains(self, start, count):
        return start >= 0 and start + count <= self.n_samples

    def read(self, start, count):
        out = np.zeros(count, dtype=np.complex64)
        a, b = max(start, 0), min(start + count, self.n_samples)
        if b > a:
            out[a - start:b - start] = self.meta.read_samples(start_index=a, count=b - a)
        return out


CHAN_RE = re.compile(r"chan([01])")


class PolPair:
    """
    Two linear polarisations (chan0, chan1) of one recording, combined coherently to circular sample by
    sample, as in cw_detect: (x0 + w1 exp(j pol_phase_deg) x1) / sqrt(2). Behaves like a Recording.
    With equalize_power, w1 = sqrt(P0 / P1) scales chan1 to the power of chan0 (measured over the whole
    file, noise dominated), else w1 = 1.
    """

    def __init__(self, path0, path1, pol_phase_deg, equalize_power=False):
        self.r0, self.r1 = Recording(path0), Recording(path1)
        for name in ("sample_rate", "start_time", "sample_start"):
            if getattr(self.r0, name) != getattr(self.r1, name):
                raise ValueError("%s differs between %s and %s" % (name, recording_label(path0), recording_label(path1)))
        self.meta = self.r0.meta
        self.sample_rate, self.start_time = self.r0.sample_rate, self.r0.start_time
        self.sample_start, self.center_freq = self.r0.sample_start, self.r0.center_freq
        self.n_samples = min(self.r0.n_samples, self.r1.n_samples)
        self.w1 = self.power_ratio() ** -0.5 if equalize_power else 1.0
        self.rot = np.complex64(self.w1 * np.exp(1j * np.deg2rad(pol_phase_deg)))

    def power_ratio(self, n_chunks=20, chunk_s=1.0):
        """P1 / P0, from n_chunks chunks of chunk_s spread over the file."""
        count = min(int(chunk_s * self.sample_rate), self.n_samples)
        p = np.zeros(2)
        for s in np.linspace(0, self.n_samples - count, n_chunks).astype(np.int64):
            p += [np.mean(np.abs(r.read(int(s), count)) ** 2) for r in (self.r0, self.r1)]
        return p[1] / p[0]

    sample_index = Recording.sample_index
    contains = Recording.contains

    def read(self, start, count):
        return ((self.r0.read(start, count) + self.rot * self.r1.read(start, count)) / np.sqrt(2)).astype(np.complex64)


# ---------------------------------------------------------------------------------------------
# Transmitter
# ---------------------------------------------------------------------------------------------

def tone_samples(p, tone, n0, count):
    """count samples of a tone starting at absolute sample n0 (exact integer phase, continuous)."""
    b = int(p.tone_bins[tone]) % p.frame_samples
    n = np.arange(n0, n0 + count, dtype=np.int64)
    return np.exp(2j * np.pi * ((b * n) % p.frame_samples) / p.frame_samples).astype(np.complex64)


def scramble_pn(p, tx_index=0):
    """
    Per-dwell scrambling mask for transmission tx_index (zeros when scrambling is off).

    Derived from SHA-256 of (seed, tx index, dwell), so it never changes with library versions.
    XOR is addition in GF(M): tone = symbol XOR pn, symbol = tone XOR pn.
    """
    if not p.scramble:
        return np.zeros(p.n_dwells, dtype=np.int64)
    pn = np.empty(p.n_dwells, dtype=np.int64)
    for d in range(p.n_dwells):
        h = hashlib.sha256(b"venus-mfsk:%d:%d:%d" % (p.scramble_seed, tx_index, d)).digest()
        pn[d] = int.from_bytes(h[:8], "big") & (p.M - 1)
    return pn


def tx_tones(p, codeword, tx_index=0):
    """Tone index of every dwell of transmission tx_index."""
    return np.asarray(codeword)[p.dwell_symbol(np.arange(p.n_dwells))] ^ scramble_pn(p, tx_index)


TX_INDEX_RE = re.compile(r"-tx(\d+)$")


def tx_set_name(name):
    """Tx file name without directory, .sigmf-meta and -txNN suffix: the name of its scramble set."""
    base = os.path.basename(name)
    base = base[:-len(".sigmf-meta")] if base.endswith(".sigmf-meta") else base
    return TX_INDEX_RE.sub("", base)


def tx_index_of(meta):
    """Tx index of a tx file: mfsk:tx_index from its metadata, else from the -txNN suffix, else 0."""
    g = read_global(meta)
    if "mfsk:tx_index" in g:
        return int(g["mfsk:tx_index"])
    m = TX_INDEX_RE.search(os.path.basename(meta_path(meta))[:-len(".sigmf-meta")])
    return int(m.group(1)) if m else 0


def tx_set_files(meta):
    """All tx meta files of the scramble set that meta belongs to, sorted by tx index (just meta if unscrambled)."""
    p = MFSKParams.from_sigmf(meta_path(meta))
    if not p.scramble:
        return [meta_path(meta)]
    d = os.path.dirname(meta_path(meta))
    files = glob.glob(os.path.join(d, glob.escape(tx_set_name(meta)) + "-tx*.sigmf-meta"))
    files = [f for f in files if TX_INDEX_RE.search(f[:-len(".sigmf-meta")])]
    return sorted(files, key=tx_index_of)


def check_tx_overwrite(base, p, tx_index=0):
    """Refuse to overwrite a tx file whose waveform parameters differ: recordings naming it would be decoded wrongly."""
    if not os.path.exists(meta_path(base)):
        return
    old = MFSKParams.from_sigmf(meta_path(base))
    diff = ["%s %r -> %r" % (f.name, getattr(old, f.name), getattr(p, f.name))
            for f in fields(MFSKParams) if getattr(old, f.name) != getattr(p, f.name)]
    old_index = tx_index_of(base)
    if p.scramble and old_index != tx_index:
        diff.append("tx_index %d -> %d" % (old_index, tx_index))
    if diff:
        raise FileExistsError("%s exists with different parameters (%s); use another name or force=True"
                              % (meta_path(base), ", ".join(diff)))


def write_tx(base, text, p, codec=None, ramp_s=0.01, backoff_db=1.0, author="", tx_gain=None,
             datetime_iso=None, force=False, tx_index=0):
    """
    Generate the transmit signal and write it as a ci16_le SigMF recording. Returns (meta path, codeword, peak).
    An existing file with the same name is only overwritten if its waveform parameters are identical, or force=True.
    """
    if not force:
        check_tx_overwrite(base, p, tx_index)
    codec = codec or Codec(p)
    codeword = codec.encode(text)
    tones = tx_tones(p, codeword, tx_index)
    scale = (2**15 - 1) * 10**(-backoff_db / 20)
    n_ramp = int(round(ramp_s * p.sample_rate))
    ramp = (0.5 - 0.5 * np.cos(np.pi * (np.arange(n_ramp) + 0.5) / n_ramp)).astype(np.float32)

    os.makedirs(os.path.dirname(os.path.abspath(data_path(base))), exist_ok=True)
    peak = 0
    with open(data_path(base), "wb") as f:
        for d, tone in enumerate(tones):
            x = tone_samples(p, tone, d * p.dwell_samples, p.dwell_samples)
            if d == 0 and n_ramp:
                x[:n_ramp] *= ramp
            if d == p.n_dwells - 1 and n_ramp:
                x[-n_ramp:] *= ramp[::-1]
            peak = max(peak, write_ci16(f, x, scale))

    extensions = [{"name": n, "version": "1.0.0", "optional": True} for n in ("vrt", "tx", "mfsk")]
    info = {
        SigMFFile.SAMPLE_RATE_KEY: p.sample_rate,
        SigMFFile.AUTHOR_KEY: author,
        SigMFFile.DESCRIPTION_KEY: "Venus MFSK tx: M=%d, RS(%d,%d), %.1f s" % (p.M, p.rs_n, p.rs_k, p.tx_actual_s),
        SigMFFile.EXTENSIONS_KEY: extensions,
        "tx:backoff_db": backoff_db,
        "tx:ramp_s": ramp_s,
        "mfsk:message": text.upper().ljust(p.n_chars),
        "mfsk:tx_index": tx_index,
        **p.to_sigmf(),
    }
    if tx_gain is not None:
        info["vrt:tx_gain"] = tx_gain
    path = write_meta(base, info, p.center_freq, datetime_iso or utc_now_iso())
    return path, codeword, int(peak)


def write_tx_set(base, text, p, n_tx=10, **kw):
    """
    Write the tx file(s): one file when scrambling is off, else n_tx files base-tx00 ... with their own
    scrambling (tx index in the name and in mfsk:tx_index). Returns [(meta path, peak int16)], codeword.
    """
    codec = kw.pop("codec", None) or Codec(p)
    if not p.scramble:
        path, codeword, peak = write_tx(base, text, p, codec=codec, **kw)
        return [(path, peak)], codeword
    out = []
    for i in range(n_tx):
        path, codeword, peak = write_tx("%s-tx%02d" % (base, i), text, p, codec=codec, tx_index=i, **kw)
        out.append((path, peak))
    return out, codeword


# ---------------------------------------------------------------------------------------------
# Channel simulator
# ---------------------------------------------------------------------------------------------

@dataclass
class ChannelParams:
    prn0_dbhz: float = 10.0         # total echo power to noise density (dB-Hz)
    core_fwhm_hz: float = 1.25      # Gaussian quasi-specular core (optimal detection bin ~1.2 x FWHM = 1.5 Hz)
    skirt_fraction: float = 0.2     # fraction of the echo power in the diffuse skirt
    skirt_limb_hz: float = 22.0     # limb Doppler of the skirt (half the limb-to-limb width)
    doppler_spread: bool = True     # False: echo is an attenuated copy of the tx signal (no fading)
    freq_offset_hz: float = 0.0     # residual Doppler offset
    noise_rms_dbfs: float = -30.0   # noise RMS in the output file
    lead_s: float = 5.0             # noise before the echo
    tail_s: float = 5.0             # noise after the echo
    fading_rate_hz: float = 200.0   # sample rate of the generated fading process


def echo_psd(f, ch):
    """Echo Doppler power spectrum (integrates to 1): Gaussian core + (1 - (f/limb)^2) diffuse skirt."""
    f = np.asarray(f, dtype=np.float64)
    sigma = ch.core_fwhm_hz / (2 * np.sqrt(2 * np.log(2)))
    core = np.exp(-0.5 * (f / sigma)**2) / (sigma * np.sqrt(2 * np.pi))
    x = f / ch.skirt_limb_hz
    skirt = np.where(np.abs(x) < 1, 1 - x**2, 0) * 3 / (4 * ch.skirt_limb_hz)
    return (1 - ch.skirt_fraction) * core + ch.skirt_fraction * skirt


def fading_process(n_samples, fs, ch, rng):
    """Complex Gaussian process (E|z|^2 = 1) with PSD echo_psd, at rate ch.fading_rate_hz. Returns (z, rate)."""
    fz = ch.fading_rate_hz
    nz = int(np.ceil(n_samples / fs * fz)) + 2
    w = (rng.standard_normal(nz) + 1j * rng.standard_normal(nz)) / np.sqrt(2)
    f = np.fft.fftfreq(nz, 1 / fz)
    H = np.sqrt(echo_psd(f, ch) * fz)          # |H|^2 averages to 1 over the band
    z = np.fft.ifft(np.fft.fft(w) * H)
    z /= np.sqrt(np.mean(np.abs(z)**2))
    return z, fz


def simulate_echo(tx_base, out_base, ch, seed=None, rec_start_iso=None, chunk_s=1.0):
    """
    Echo of a tx recording through the Venus channel, written as a ci16_le SigMF recording.
    Returns (rx meta path, echo start ISO timestamp).
    """
    rng = np.random.default_rng(seed)
    tx = Recording(tx_base)
    fs = tx.sample_rate
    n_tx = tx.n_samples
    n_lead, n_tail = int(round(ch.lead_s * fs)), int(round(ch.tail_s * fs))
    n_total = n_lead + n_tx + n_tail

    # normalize the tx to unit amplitude (constant envelope), measured away from the ramps
    probe = tx.read(n_tx // 2, min(n_tx // 2, int(fs)))
    tx_amp = np.sqrt(np.mean(np.abs(probe)**2))

    scale = 2**15 - 1
    noise_rms = 10**(ch.noise_rms_dbfs / 20)             # complex RMS, full scale = 1
    n0 = noise_rms**2 / fs                                # noise density per Hz
    sig_amp = np.sqrt(10**(ch.prn0_dbhz / 10) * n0)

    if ch.doppler_spread:
        z, fz = fading_process(n_tx, fs, ch, rng)
        zi = np.arange(len(z))

    rec_start = parse_iso(rec_start_iso or utc_now_iso())
    echo_start = rec_start + np.timedelta64(int(round(n_lead / fs * 1e9)), "ns")

    os.makedirs(os.path.dirname(os.path.abspath(data_path(out_base))), exist_ok=True)
    chunk = int(chunk_s * fs)
    with open(data_path(out_base), "wb") as f:
        for a in range(0, n_total, chunk):
            b = min(a + chunk, n_total)
            y = ((rng.standard_normal(b - a, dtype=np.float32)
                  + 1j * rng.standard_normal(b - a, dtype=np.float32)) * (noise_rms / np.sqrt(2)))
            ta, tb = max(a, n_lead), min(b, n_lead + n_tx)
            if tb > ta:
                x = tx.read(ta - n_lead, tb - ta) / tx_amp
                t = np.arange(ta - n_lead, tb - n_lead) / fs
                g = sig_amp * np.exp(2j * np.pi * ch.freq_offset_hz * t)
                if ch.doppler_spread:
                    g = g * (np.interp(t * fz, zi, z.real) + 1j * np.interp(t * fz, zi, z.imag))
                y[ta - a:tb - a] += (x * g).astype(np.complex64)
            write_ci16(f, y, scale)

    info = {
        SigMFFile.SAMPLE_RATE_KEY: fs,
        # same form as the real recordings, '<tx file>;<tx gain>;<tx unix time>;<object>', so rx can scan it
        SigMFFile.DESCRIPTION_KEY: "%s;0;%.6f;sim" % (
            os.path.basename(meta_path(tx_base)),
            (echo_start - np.datetime64(0, "ns")).astype("timedelta64[ns]").astype(np.int64) * 1e-9),
        SigMFFile.EXTENSIONS_KEY: [{"name": "sim", "version": "1.0.0", "optional": True}],
        **{"sim:" + k: v for k, v in asdict(ch).items()},
        "sim:seed": seed,
    }
    path = write_meta(out_base, info, tx.center_freq, format_iso(rec_start),
                      annotations=[(n_lead, n_tx, {SigMFFile.LABEL_KEY: "echo"})])
    return path, format_iso(echo_start)


# ---------------------------------------------------------------------------------------------
# Receiver
# ---------------------------------------------------------------------------------------------

@dataclass
class RxParams:
    kernel_bins: int = 1            # odd; detection bins summed around each tone
    freq_offset_hz: float = 0.0     # residual Doppler to remove before the FFT
    window: str = "hann"            # "hann": WOLA, Hann windows with 50% overlap inside each dwell; "rect": plain frames
    pol_phase_deg: float = 265.0    # chan0/chan1 pairs: (x0 + w1 exp(j phase) x1) / sqrt(2), as in cw_detect
    equalize_power: bool = False    # chan0/chan1 pairs: w1 = sqrt(P0 / P1) scales chan1 to the power of chan0; False: w1 = 1
    rfi_window_dwells: int = 2      # interference flagging: dwells on each side used as background (0 = off)
    rfi_threshold: float = 5.0      # flag a tone in a dwell if its background exceeds this x the noise mean


def frame_plan(p, window="hann"):
    """
    Analysis frames of one transmission: (start sample from the echo start, dwell index, window or None,
    noise variance factor of a dwell sum).

    "rect": the n_frames back-to-back frames. "hann": Hann windows with 50% overlap, only those entirely
    inside one dwell (a window across a dwell boundary would mix two tones). Overlapping windows are
    slightly correlated; the variance factor is the noise variance of a dwell sum relative to independent frames.
    """
    Nf = p.frame_samples
    if window == "rect":
        starts = np.arange(p.n_frames, dtype=np.int64) * Nf
        return starts, starts // p.dwell_samples, None, 1.0
    if window != "hann":
        raise ValueError("window must be 'hann' or 'rect'")
    hop = Nf // 2
    J = (p.dwell_samples - Nf) // hop + 1                       # windows per dwell
    starts = (np.arange(p.n_dwells, dtype=np.int64)[:, None] * p.dwell_samples
              + np.arange(J, dtype=np.int64)[None, :] * hop).ravel()
    dwell = np.repeat(np.arange(p.n_dwells), J)
    w = (0.5 - 0.5 * np.cos(2 * np.pi * np.arange(Nf) / Nf)).astype(np.float32)
    c = (np.sum(w[:Nf - hop] * w[hop:]) / np.sum(w**2))**2      # power correlation of adjacent windows
    return starts, dwell, w, (J + 2 * (J - 1) * c) / J


def flag_interference(D, n, W, thr):
    """
    Flag persistent narrowband interference in dwell energies D (n_dwells x M, noise mean n[d] per dwell).

    A tone is flagged in dwell d when the median normalized energy of that tone in the W dwells before d,
    or in the W dwells after d, exceeds thr. Our own signal moves to another tone every dwell, so it is
    not in its own background; a carrier that lasts longer than a dwell is.
    """
    ok = n > 0
    Dn = np.where(ok[:, None], D / np.maximum(n, 1)[:, None], np.nan)
    flag = np.zeros(D.shape, dtype=bool)
    for d in np.flatnonzero(ok):
        bg = np.zeros(D.shape[1])
        for side in (Dn[max(0, d - W):d], Dn[d + 1:d + 1 + W]):
            side = side[~np.isnan(side[:, 0])]
            if len(side):
                bg = np.maximum(bg, np.median(side, axis=0))
        flag[d] = bg > thr
    return flag


def band_bins(p):
    """FFT bins covered by the tone block, used for the per-frame noise estimate."""
    return np.arange(p.tone_bins.min(), p.tone_bins.max() + p.tone_spacing_bins) % p.frame_samples


def frame_power(rec, a, p, rx, w, n0):
    """Noise-normalized power spectrum of the frame starting at recording sample a (n0: sample index from echo start)."""
    Nf = p.frame_samples
    x = rec.read(a, Nf)
    if rx.freq_offset_hz:
        n = np.arange(n0, n0 + Nf)
        x = x * np.exp(-2j * np.pi * rx.freq_offset_hz * n / p.sample_rate).astype(np.complex64)
    if w is not None:
        x = x * w
    P = np.abs(np.fft.fft(x))**2
    return P / max(np.median(P[band_bins(p)]) / np.log(2), 1e-30)   # exponential noise: mean = median / ln 2


class Accumulator:
    """Noise-normalized tone energies, summed per code symbol over all frames and recordings."""

    def __init__(self, p, rx=None):
        self.p, self.rx = p, rx or RxParams()
        self.E = np.zeros((p.rs_n, p.M))
        self.L = np.zeros(p.rs_n)       # noise mean of E per symbol
        self.V = np.zeros(p.rs_n)       # noise variance of E per symbol
        self.frames = []                # per recording: (frames, M) float32 frame energies (flagged cells at noise mean)
        self.frame_dwell = []           # per recording: dwell index of every frame
        self.frame_time = []            # per recording: frame start in s from the echo start
        self.flags = []                 # per recording: (n_dwells, M) interference flags
        self.tx_index = []              # per recording: tx index (scrambling)
        self.labels = []

    def add_recording(self, path, echo_start_iso, tx_index=0, progress=True):
        """path: a SigMF recording, or a (chan0, chan1) pair combined to circular polarisation."""
        p, rx = self.p, self.rx
        rec = open_recording(path, rx.pol_phase_deg, rx.equalize_power)
        if rec.sample_rate != p.sample_rate:
            raise ValueError("sample rate %g != %g" % (rec.sample_rate, p.sample_rate))
        start = rec.sample_index(echo_start_iso)
        Nf = p.frame_samples
        h = rx.kernel_bins // 2
        idx = (p.tone_bins[:, None] + np.arange(-h, h + 1)[None, :]) % Nf
        starts, fdwell, w, var = frame_plan(p, rx.window)
        fe = np.full((len(starts), p.M), rx.kernel_bins, dtype=np.float32)  # noise mean for skipped frames
        used = np.zeros(len(starts), dtype=bool)
        for j, s0 in enumerate(starts):
            if not rec.contains(start + s0, Nf):
                continue                                    # frame not (fully) inside the recording
            used[j] = True
            fe[j] = frame_power(rec, start + s0, p, rx, w, s0)[idx].sum(axis=1)
            if progress and j % 50 == 0:
                print("\r%s: frame %d/%d" % (recording_label(path), j, len(starts)), end="")

        # per-dwell energies, with persistent interference replaced by the noise mean
        n = np.bincount(fdwell[used], minlength=p.n_dwells) * rx.kernel_bins   # noise mean per dwell
        D = np.zeros((p.n_dwells, p.M))
        np.add.at(D, fdwell[used], fe[used])
        if rx.rfi_window_dwells > 0:
            flag = flag_interference(D, n, rx.rfi_window_dwells, rx.rfi_threshold)
        else:
            flag = np.zeros(D.shape, dtype=bool)
        D = np.where(flag, n[:, None], D)
        fe[flag[fdwell]] = rx.kernel_bins
        # descramble: symbol value v was sent on tone v XOR pn[d] (identity when scrambling is off)
        pn = scramble_pn(p, tx_index)
        D = D[np.arange(p.n_dwells)[:, None], np.arange(p.M)[None, :] ^ pn[:, None]]
        if progress:
            print("\r%s: %d/%d frames (%s) inside the recording, echo starts at sample %d, %.2f%% tone cells flagged as RFI"
                  % (recording_label(path), used.sum(), len(starts), rx.window, start, 100 * flag[n > 0].mean()))
        sym = p.dwell_symbol(np.arange(p.n_dwells))
        np.add.at(self.E, sym, D)
        np.add.at(self.L, sym, n)
        np.add.at(self.V, sym, n * var)
        self.frames.append(fe)
        self.frame_dwell.append(fdwell)
        self.frame_time.append(starts / p.sample_rate)
        self.flags.append(flag)
        self.tx_index.append(tx_index)
        self.labels.append(recording_label(path))

    @property
    def z(self):
        """Per-symbol z-scores (E - L) / sqrt(V)."""
        return (self.E - self.L[:, None]) / np.sqrt(self.V[:, None])


def echo_profile(recordings, p, codeword, halfwidth_bins=25, return_count=False, rx=None):
    """
    Average spectrum around the transmitted tone, over all frames (inside the recordings) and
    recordings, noise-normalized, with the receiver's window. Needs the codeword (known or decoded).
    recordings: (path, echo start) or (path, echo start, tx index) tuples.
    Returns (freq offsets Hz, profile), plus the effective number of independent frames if return_count.
    """
    rx = rx or RxParams()
    Nf = p.frame_samples
    offs = np.arange(-halfwidth_bins, halfwidth_bins + 1)
    starts, fdwell, w, var = frame_plan(p, rx.window)
    acc = np.zeros(len(offs))
    n = 0
    for path, echo_start, *rest in recordings:
        tones = tx_tones(p, codeword, rest[0] if rest else 0)
        rec = open_recording(path, rx.pol_phase_deg, rx.equalize_power)
        start = rec.sample_index(echo_start)
        for s0, d in zip(starts, fdwell):
            if not rec.contains(start + s0, Nf):
                continue
            P = frame_power(rec, start + s0, p, rx, w, s0)
            acc += P[(p.tone_bins[tones[d]] + offs) % Nf]
            n += 1
    if n == 0:
        raise ValueError("no frames inside the recordings")
    if return_count:
        return offs * p.bin_actual_hz, acc / n, n / var
    return offs * p.bin_actual_hz, acc / n


# ---------------------------------------------------------------------------------------------
# Recording directories
# ---------------------------------------------------------------------------------------------

def parse_description(desc):
    """
    '<tx file>;<tx gain>;<tx unix time>;<object>' -> dict, or None if the description does not start with
    a tx file name. The fields after the tx file are optional (missing or unparsable ones become None),
    so a description with only the tx file name (with or without .sigmf-meta) is accepted too.
    """
    parts = [s.strip() for s in (desc or "").split(";")]
    name = parts[0]
    if not name or any(c.isspace() for c in name):
        return None                                 # free text such as "Venus EVE 2025"

    def field(i, conv):
        try:
            return conv(parts[i]) if len(parts) > i and parts[i] else None
        except ValueError:
            return None

    return {"tx_file": name if name.endswith(".sigmf-meta") else name + ".sigmf-meta",
            "tx_gain": field(1, float), "tx_time": field(2, float), "object": field(3, str)}


def pair_polarisations(found, skipped):
    """Merge chan0/chan1 recordings with otherwise identical names into one entry with path = (chan0, chan1)."""
    groups = {}
    for d in found:
        name = os.path.basename(d["path"])
        key = CHAN_RE.sub("chan*", name) if CHAN_RE.search(name) else name
        groups.setdefault(key, []).append(d)
    out = []
    for key, ds in groups.items():
        chans = {CHAN_RE.search(os.path.basename(d["path"])).group(1): d for d in ds
                 if CHAN_RE.search(os.path.basename(d["path"]))}
        if len(ds) == 2 and set(chans) == {"0", "1"}:
            d0, d1 = chans["0"], chans["1"]
            if ((d0["tx_file"], d0["tx_time"], d0.get("radar_echo_time"))
                    != (d1["tx_file"], d1["tx_time"], d1.get("radar_echo_time"))):
                skipped += [(d["path"], "chan0/chan1 descriptions differ") for d in (d0, d1)]
                continue
            d = dict(d0, path=(d0["path"], d1["path"]))
            d["rx_gain"] = (d0["rx_gain"], d1["rx_gain"]) if d0["rx_gain"] != d1["rx_gain"] else d0["rx_gain"]
            out.append(d)
        else:
            out += ds                       # single channel, or no chan0/chan1 pair: use the files as they are
    return out


def rx_meta_files(rx):
    """
    The *.sigmf-meta files in directory rx, or the files in the list rx (with or without .sigmf-meta). A listed
    chan0 file brings its chan1 partner and vice versa, when that exists.
    """
    if isinstance(rx, str) and os.path.isdir(rx):
        return sorted(glob.glob(os.path.join(rx, "*.sigmf-meta")))
    files = []
    for f in [rx] if isinstance(rx, str) else rx:
        f = meta_path(f)
        if not os.path.exists(f):
            raise FileNotFoundError(f)
        partner = os.path.join(os.path.dirname(f), CHAN_RE.sub(lambda m: "chan%d" % (1 - int(m.group(1))),
                                                               os.path.basename(f)))
        files += [g for g in (f, partner) if g not in files and os.path.exists(g)]
    return files


NOT_MFSK = "not an MFSK recording"      # start of the skip reason for recordings of other transmissions


def scan_recordings(rx_dir, tx_dir, tx_name=None, echo_delay_s=0.0, combine_pol=True):
    """
    Find the SigMF recordings of an MFSK transmission, in directory rx_dir or in a list of files (see rx_meta_files).
    A recording's core:description is '<tx file>;<tx gain>;<tx unix time>;<object>'; only the tx file is required.
    It is an MFSK recording when its tx file in tx_dir has mfsk:* fields (or, if the tx file is not there, has
    mfsk in its name); recordings of other transmissions and recordings whose tx file is not in tx_dir are skipped.
    Simulated echoes (sim_mfsk, sim:* fields) get "sim" = True.

    The tx files are looked up in tx_dir. All recordings must belong to one tx set: the same tx file, or
    files of one scramble set (name-txNN) with identical parameters. tx_name (a tx file or set name)
    selects one set when the directory holds several. Echo start, plus echo_delay_s: radar:echo_time (tx time +
    round trip, written by sigmf_dedoppler.py) when the recording has it, else the tx unix time from the
    description, else radar:tx_time, else the start of the recording.
    With combine_pol, files whose names differ only in chan0/chan1 are one recording of two polarisations:
    their "path" is the (chan0, chan1) pair, combined coherently by the receiver.
    Returns (tx meta path of the first recording, recordings sorted by tx time as dicts with tx_index,
    skipped files as (path, reason)).
    """
    found, skipped = [], []
    for path in rx_meta_files(rx_dir):
        try:
            with open(path) as f:
                meta = json.load(f)
        except (OSError, ValueError) as e:
            skipped.append((path, "unreadable: %s" % e))
            continue
        g = meta.get("global", {})
        if "mfsk:M" in g:
            skipped.append((path, NOT_MFSK + ": an MFSK tx file"))
            continue
        desc = g.get(SigMFFile.DESCRIPTION_KEY, "")
        d = parse_description(desc)
        if d is None:
            skipped.append((path, "description %r" % desc))
            continue
        tx_path = os.path.join(tx_dir, d["tx_file"])
        if not (os.path.exists(tx_path) and "mfsk:M" in read_global(tx_path)
                or not os.path.exists(tx_path) and "mfsk" in d["tx_file"].lower()):
            skipped.append((path, NOT_MFSK + ": tx file %s" % d["tx_file"]))
            continue
        if tx_name is not None and tx_set_name(d["tx_file"]) != tx_set_name(tx_name):
            skipped.append((path, "tx file %s" % d["tx_file"]))
            continue
        if not os.path.exists(tx_path):
            skipped.append((path, "tx file %s not in %s" % (d["tx_file"], tx_dir)))
            continue
        d["path"] = path
        d["sim"] = any(k.startswith("sim:") for k in g)
        d["rx_gain"] = g.get("vrt:rx_gain")
        d["radar_echo_time"] = g.get("radar:echo_time")      # set by sigmf_dedoppler.py
        d["radar_tx_time"] = g.get("radar:tx_time")
        d["capture_start"] = (meta.get("captures") or [{}])[0].get(SigMFFile.DATETIME_KEY)
        found.append(d)

    if combine_pol:
        found = pair_polarisations(found, skipped)

    if not found:
        why = "".join("\n  %s: %s" % (os.path.basename(p_), r) for p_, r in skipped) or " (no *.sigmf-meta files)"
        raise ValueError("no usable recordings in %s%s%s" % (
            rx_dir, "" if tx_name is None else " for tx_name %r" % tx_name, why))
    sets = sorted({tx_set_name(d["tx_file"]) for d in found})
    if len(sets) > 1:
        raise ValueError("recordings belong to different tx sets %s; select one with tx_name" % sets)

    params = None
    for d in found:
        d["tx_path"] = os.path.join(tx_dir, d["tx_file"])
        p = MFSKParams.from_sigmf(meta_path(d["tx_path"]))
        if params is None:
            params = p
        elif p != params:
            raise ValueError("tx files of set %s have different parameters (%s vs %s)"
                             % (sets[0], d["tx_file"], found[0]["tx_file"]))
        d["tx_index"] = tx_index_of(d["tx_path"])

    delay = np.timedelta64(int(round(echo_delay_s * 1e9)), "ns")
    for d in found:
        if d["tx_time"] is not None:
            t = np.datetime64(int(round(d["tx_time"] * 1e9)), "ns")    # unix time -> UTC
        elif d.get("radar_tx_time"):
            t = parse_iso(d["radar_tx_time"])
        else:
            t = None
        d["tx_start"] = format_iso(t) if t is not None else None
        if d.get("radar_echo_time"):
            echo, d["echo_source"] = parse_iso(d["radar_echo_time"]), "radar:echo_time"
        elif d["tx_time"] is not None:
            echo, d["echo_source"] = t, "tx time"
        elif t is not None:
            echo, d["echo_source"] = t, "radar:tx_time"
        elif d.get("capture_start"):
            echo, d["echo_source"] = parse_iso(d["capture_start"]), "recording start"
        else:
            raise ValueError("no time for the echo start in %s" % recording_label(d["path"]))
        d["echo_start"] = format_iso(echo + delay)
    found.sort(key=lambda d: d["echo_start"])
    return meta_path(found[0]["tx_path"]), found, skipped


# ---------------------------------------------------------------------------------------------
# Fast energy-domain simulation (for decoder tests and Monte Carlo)
# ---------------------------------------------------------------------------------------------

def simulate_energies(p, codeword, prn0_dbhz, n_echoes=1, capture=0.6, rng=None):
    """
    Accumulated energy matrix without generating samples. Per frame, the echo power captured in the
    tone's bin is a Rayleigh-fading fraction `capture` of Pr; independent fading per frame, so the
    signal bin sums to Gamma(L, 1 + a2) and noise bins to Gamma(L, 1).
    """
    rng = rng or np.random.default_rng()
    a2 = 10**(prn0_dbhz / 10) / p.bin_actual_hz * capture
    L = p.dwells_per_symbol * p.dwell_frames * n_echoes
    E = rng.gamma(L[:, None], 1.0, size=(p.rs_n, p.M))
    E[np.arange(p.rs_n), codeword] = rng.gamma(L, 1 + a2)
    return E, L.astype(np.float64)
