Venus MFSK receiver (simple version)¶
Decodes the Venus MFSK message from raw SigMF recordings.
Inputs. The recordings are all *.sigmf-meta files in rx_dir, or the files in rx_files (rx_dir = None); a listed chan0 file brings its chan1 partner. Each has a core:description of the form <tx file>;<tx gain>;<tx unix time>;<target>. The tx file is looked up in tx_dir, and its mfsk:* fields give the waveform parameters and the tx index (scrambling). Recordings whose tx file is not an MFSK file (CW, Q65, ZC, ...) are skipped. Simulated echoes from sim_mfsk (sim:* fields) have no Doppler; their echo starts at the tx unix time. Files whose names differ only in chan0/chan1 are two linear polarisations of one recording; they are combined to circular as (x0 + exp(j phase) x1) / sqrt(2). The receiver site comes from the file name (stockert_..., dwingeloo_..., wsrt_...).
Processing per recording
- Doppler: SPICE gives the Doppler of the sub-radar point once per second. It is spline-interpolated, integrated to a phase and removed from the samples. SPICE also gives the echo start: the tx time plus the round-trip light time.
- Each dwell (one tone) is split into Hann windows of one frame (1 FFT bin = 1.5 Hz) with 50% overlap. Every spectrum is divided by its noise mean (median over the tone block / ln 2). The energy at each tone is summed per dwell.
- RFI: the tone changes every dwell, but a carrier doesn't. A tone whose energy stays high in the neighbouring dwells is flagged and set to the noise mean.
- Descrambling (
tone = symbol XOR pn) and accumulation per code symbol intoE(n x M), together with its noise meanLand varianceV.
Decoding runs after every recording: ordered-statistics decoding over the RS code, then a CRC check.
import glob
import hashlib
import itertools
import json
import os
import re
from types import SimpleNamespace
import galois
import numpy as np
import scipy.interpolate
import spiceypy as sp
Parameters¶
rx_dir = "/Users/telkamp/data/eve2026/wsrt_mfsk2" # raw recordings (not Doppler-corrected)
# rx_dir = "data" # sim_mfsk output
# rx_dir = None # use rx_files; a chan0 file brings its chan1
rx_files = [
"/Users/telkamp/data/eve2026/wsrt_mfsk2/wsrt_venus_2026_10_18_11_00_45_1299.650MHz_0.25Msps_ci16_le.chan0.sigmf-meta",
]
tx_dir = "/Users/telkamp/data/eve2026/tx_signals_eve/mfsk-v1" # tx files named in core:description
kernel_dir = "/Users/telkamp/notebooks/kernels" # SPICE kernels
tx_site = "DWINGELOO" # transmitter (SPICE name)
pol_phase_deg = 210.0 # chan0/chan1 pairs: (x0 + exp(j phase) x1) / sqrt(2)
rfi_dwells = 2 # RFI flagging: background from this many dwells on each side
rfi_threshold = 2 # flag a tone in a dwell if its background exceeds this x the noise mean
osd = dict(n_bases=256, list_depth=10, n_check=50, n_free_sets=16, free_depth=5) # decoder effort
Waveform and message format¶
The tx file's mfsk:* fields define the waveform:
- frame =
1 / bin_hz; a dwell isdwell_framesframes on one tone - tone
tsits on FFT bin(t - M/2) * tone_spacing_bins - dwell
dcarries code symbold mod n
The message is a base-37 number followed by a CRC, cut into k symbols of log2 M bits. Those k symbols are then encoded with RS(n, k) over GF(M).
ALPHABET = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" # space = 0
CRC_SPECS = {16: (0x1021, 0xFFFF), 24: (0x864CFB, 0xB704CE)} # width -> (polynomial, init)
def read_meta(path):
with open(path) as f:
return json.load(f)
def mfsk_params(tx_meta):
"""Waveform parameters from the mfsk:* fields of a tx .sigmf-meta file."""
g = read_meta(tx_meta)["global"]
p = SimpleNamespace(fs=g["mfsk:sample_rate"], M=g["mfsk:M"], n=g["mfsk:rs_n"], n_chars=g["mfsk:n_chars"],
crc_bits=g["mfsk:crc_bits"], scramble=g["mfsk:scramble"], seed=g["mfsk:scramble_seed"])
spacing = g["mfsk:tone_spacing_bins"]
p.Nf = int(round(p.fs / g["mfsk:bin_hz"])) # frame = FFT length
p.dwell = p.Nf * g["mfsk:dwell_frames"] # samples per dwell
p.n_dwells = int(g["mfsk:tx_length_s"] * p.fs // p.dwell)
p.tone_bins = ((np.arange(p.M) - p.M // 2) * spacing) % p.Nf # FFT bin of every tone
p.band = np.arange(-p.M // 2 * spacing, p.M // 2 * spacing) % p.Nf # all bins of the tone block
p.bits = p.M.bit_length() - 1 # bits per code symbol
p.msg_bits = (len(ALPHABET) ** p.n_chars - 1).bit_length()
p.k = -(-(p.msg_bits + p.crc_bits) // p.bits) # information symbols
return p
def scramble_pn(p, tx_index):
"""Scrambling mask per dwell: tone = symbol XOR pn[dwell], from SHA-256 of (seed, tx index, dwell)."""
if not p.scramble:
return np.zeros(p.n_dwells, dtype=np.int64)
return np.array([int.from_bytes(hashlib.sha256(b"venus-mfsk:%d:%d:%d" % (p.seed, tx_index, d)).digest()[:8], "big")
& (p.M - 1) for d in range(p.n_dwells)])
def crc(value, nbits, width):
poly, reg = CRC_SPECS[width]
for i in reversed(range(nbits)):
top = (reg >> (width - 1)) & 1
reg = (reg << 1) & ((1 << width) - 1)
if top ^ ((value >> i) & 1):
reg ^= poly
return reg
def symbols_to_text(info, p):
"""k information symbols -> message, or None if the base-37 range or the CRC check fails."""
word = 0
for s in info:
word = (word << p.bits) | int(s)
value, check = word >> p.crc_bits, word & ((1 << p.crc_bits) - 1)
if value >= len(ALPHABET) ** p.n_chars or crc(value, p.msg_bits, p.crc_bits) != check:
return None
text = ""
for _ in range(p.n_chars):
value, i = divmod(value, len(ALPHABET))
text = ALPHABET[i] + text
return text
Recordings¶
Every recording in rx_dir (or rx_files) whose description names an MFSK tx file in tx_dir, with chan0/chan1 files paired. Sorted by tx time.
RX_SITES = {"dwingeloo": "DWINGELOO", "stockert": "STOCKERT", "wsrt": "WSRT_RT2"} # file name prefix -> SPICE site
TARGETS = ("moon", "venus")
CHAN_RE = re.compile(r"chan[01]")
TX_SET_RE = re.compile(r"-tx\d+\.sigmf-meta$")
def rx_meta_files(rx):
"""*.sigmf-meta files in directory rx, or the files in list rx plus the chan1 (chan0) partner of a chan0 (chan1) file."""
if isinstance(rx, str):
return sorted(glob.glob(os.path.join(rx, "*.sigmf-meta")))
files = []
for f in rx:
partner = os.path.join(os.path.dirname(f), CHAN_RE.sub(lambda m: "chan1" if m.group() == "chan0" else "chan0",
os.path.basename(f)))
files += [g for g in (f, partner) if g not in files and os.path.exists(g)]
return files
def scan_recordings(rx, tx_dir):
recs, n_other = {}, 0
for meta in rx_meta_files(rx):
g = read_meta(meta)["global"]
fields = g.get("core:description", "").split(";")
tx_file = fields[0] if fields[0].endswith(".sigmf-meta") else fields[0] + ".sigmf-meta"
tx_path = os.path.join(tx_dir, tx_file)
if (" " in fields[0] or "mfsk:M" in g
or ("mfsk:M" not in read_meta(tx_path)["global"] if os.path.exists(tx_path) else "mfsk" not in tx_file)):
n_other += 1 # other transmission, or a tx file
continue
sim = any(k.startswith("sim:") for k in g) # simulated echo: no Doppler
name = os.path.basename(meta).lower().split("_")
if len(fields) < 3 or not os.path.exists(tx_path) or not (sim or name[0] in RX_SITES):
print("skipped %s (description %r)" % (os.path.basename(meta), ";".join(fields)))
continue
target = next((f for f in name if f in TARGETS), fields[3].lower() if len(fields) > 3 else "")
key = CHAN_RE.sub("chan0+1", os.path.basename(meta))
r = recs.setdefault(key, dict(name=key, metas=[], tx_file=tx_file, tx_unix=float(fields[2]),
receiver=None if sim else RX_SITES[name[0]], target=target.upper()))
r["metas"].append(meta)
if n_other:
print("skipped %d files of other transmissions or tx files" % n_other)
if not recs:
raise ValueError("no MFSK recordings with a tx file in %s" % tx_dir)
recs = sorted(recs.values(), key=lambda r: r["tx_unix"])
if len({TX_SET_RE.sub("", r["tx_file"]) for r in recs}) != 1:
raise ValueError("recordings belong to different tx sets")
for r in recs:
r["tx_index"] = read_meta(os.path.join(tx_dir, r["tx_file"]))["global"].get("mfsk:tx_index", 0)
return recs
class Recording:
"""A ci16_le SigMF recording, or a chan0/chan1 pair combined to circular polarisation."""
def __init__(self, metas, pol_phase_deg):
metas = sorted(metas) # chan0 first
heads = [read_meta(m) for m in metas]
caps = [h["captures"][0] for h in heads]
assert all(h["global"]["core:datatype"] == "ci16_le" for h in heads)
assert all(c.get("core:sample_start", 0) == 0 and c["core:datetime"] == caps[0]["core:datetime"] for c in caps)
self.fs = heads[0]["global"]["core:sample_rate"]
self.f0 = caps[0]["core:frequency"]
self.start = np.datetime64(caps[0]["core:datetime"].rstrip("Z"), "ns")
self.iq = [np.memmap(m.replace(".sigmf-meta", ".sigmf-data"), dtype="<i2", mode="r") for m in metas]
self.n_samples = min(len(iq) for iq in self.iq) // 2
self.rot = np.exp(1j * np.deg2rad(pol_phase_deg))
def read(self, a, count):
x = [iq[2 * a:2 * (a + count)].astype(np.float32).view(np.complex64) for iq in self.iq]
return x[0] if len(x) == 1 else (x[0] + self.rot * x[1]) / np.sqrt(2)
recordings = scan_recordings(rx_dir if rx_dir is not None else rx_files, tx_dir)
p = mfsk_params(os.path.join(tx_dir, recordings[0]["tx_file"]))
print("%d recordings, tx set %s" % (len(recordings), TX_SET_RE.sub("", recordings[0]["tx_file"])))
print("M=%d, RS(%d,%d), frame %d samples (%.3f Hz bins), %d dwells of %d samples, scrambling %s"
% (p.M, p.n, p.k, p.Nf, p.fs / p.Nf, p.n_dwells, p.dwell, "on" if p.scramble else "off"))
2 recordings, tx set venus-mfsk4096-rs9-sp3-d4-269s M=4096, RS(9,6), frame 166667 samples (1.500 Hz bins), 101 dwells of 666668 samples, scrambling on
Doppler correction and echo time (SPICE)¶
The round-trip light time lt and its rate dlt/dt are computed for the sub-radar point: transmitter → point → receiver, at reception time. The Doppler is -f0 · dlt/dt. It is computed once per second, interpolated with a cubic spline and integrated to a phase. That phase is then removed from the samples.
for kernel in ["naif0012.tls", "de440s.bsp", "pck00011.tpc", "earth_latest_high_prec.bpc", "antennas.fk", "antennas.bsp"]:
sp.furnsh(os.path.join(kernel_dir, kernel))
def radar_srp(target, et_rx, transmitter, receiver):
"""(d(round-trip light time)/dt, round-trip light time) of the sub-radar point, for reception at et_rx."""
frame = "IAU_" + target
srp, _, _ = sp.subpnt("INTERCEPT/ELLIPSOID", target, et_rx, frame, "CN", receiver)
ez_r, lt_r = sp.spkcpt(srp, target, frame, et_rx, "J2000", "OBSERVER", "CN", receiver) # point -> rx
ez_d, lt_d = sp.spkcpo(transmitter, et_rx - lt_r, "J2000", "OBSERVER", "CN", srp, target, frame) # tx -> point
dlt_r = sp.dvnorm(ez_r) / sp.clight()
ez_d[3:] *= 1 - dlt_r # the bounce happens lt_r earlier: rates in the point's time
return dlt_r + sp.dvnorm(ez_d) / sp.clight(), lt_r + lt_d
class RadarDoppler:
"""Doppler of one recording, removed from its samples by phase (sample 0 = recording start)."""
def __init__(self, rec, target, transmitter, receiver):
self.target, self.transmitter, self.receiver, self.fs = target, transmitter, receiver, rec.fs
self.et0 = sp.str2et(str(rec.start))
self.t = np.arange(-2.0, np.ceil(rec.n_samples / rec.fs) + 3.0)
self.doppler = np.array([-radar_srp(target, self.et0 + t, transmitter, receiver)[0] * rec.f0 for t in self.t])
self.cycles = scipy.interpolate.CubicSpline(self.t, self.doppler).antiderivative()
def remove(self, x, a):
"""x = samples a, a+1, ... with the Doppler removed."""
cycles = np.mod(self.cycles((a + np.arange(len(x))) / self.fs) - self.cycles(0.0), 1.0)
return (x * np.exp(-2j * np.pi * cycles)).astype(np.complex64)
def echo_start(self, tx_unix):
"""Seconds from the recording start to the leading edge of the echo of a transmission at unix time tx_unix."""
et_tx = sp.str2et(str(np.datetime64(round(tx_unix * 1e9), "ns")))
et_rx = et_tx
for _ in range(10): # solve et_rx = et_tx + lt(et_rx)
et_new = et_tx + radar_srp(self.target, et_rx, self.transmitter, self.receiver)[1]
if abs(et_new - et_rx) < 1e-9:
break
et_rx = et_new
return et_new - self.et0
Demodulation¶
Per dwell: Hann windows of one frame with 50% overlap, only the windows that lie entirely inside the dwell (7 for 4 frames). The windows overlap, so their energies are slightly correlated. var_factor is the noise variance of a dwell sum relative to independent windows.
hop = p.Nf // 2
n_win = (p.dwell - p.Nf) // hop + 1 # windows per dwell
window = (0.5 - 0.5 * np.cos(2 * np.pi * np.arange(p.Nf) / p.Nf)).astype(np.float32)
c = (np.sum(window[:p.Nf - hop] * window[hop:]) / np.sum(window**2))**2 # power correlation of adjacent windows
var_factor = (n_win + 2 * (n_win - 1) * c) / n_win
def dwell_energies(rec, doppler, start):
"""Energy at every tone in every dwell (n_dwells x M) in units of the noise mean per window, and the number of
windows per dwell (0 for a dwell not entirely inside the recording). start: sample of the echo start.
doppler: RadarDoppler, or None for a simulated echo."""
D = np.zeros((p.n_dwells, p.M))
n = np.zeros(p.n_dwells)
for d in range(p.n_dwells):
a = start + d * p.dwell
if a < 0 or a + p.dwell > rec.n_samples:
continue
x = rec.read(a, p.dwell)
if doppler is not None:
x = doppler.remove(x, a)
for j in range(n_win):
P = np.abs(np.fft.fft(x[j * hop:j * hop + p.Nf] * window))**2
D[d] += P[p.tone_bins] / (np.median(P[p.band]) / np.log(2)) # exponential noise: mean = median / ln 2
n[d] = n_win
return D, n
def flag_rfi(D, n, W, threshold):
"""Flag tone d of a dwell when its median energy in the W dwells before, or the W dwells after, exceeds threshold
x the noise mean. Our signal moves to another tone every dwell; a carrier stays."""
Dn = D / np.where(n > 0, n, np.nan)[:, None]
flag = np.zeros(D.shape, dtype=bool)
for d in np.flatnonzero(n > 0):
bg = np.zeros(p.M)
for side in (Dn[max(0, d - W):d], Dn[d + 1:d + 1 + W]):
side = side[~np.isnan(side[:, 0])]
if len(side):
bg = np.maximum(bg, np.median(side, axis=0))
flag[d] = bg > threshold
return flag
Decoder¶
Ordered-statistics decoding (OSD). For every code position, keep the strongest tones; a position's reliability is the gap between its best and second-best tone. Any k positions of an RS code fix the whole codeword (MDS). So for a set of k positions, every combination of their list_depth strongest tones is re-encoded to a full codeword. Each codeword is scored by sum_i E[i, c_i], the maximum-likelihood metric for square-law detection at low SNR. The first position set is the k most reliable positions; further sets are random, weighted towards reliable positions.
If that fails, a second pass needs only k-1 positions in the lists. It fixes k-1 positions to their free_depth strongest tones and tries all M values of the remaining information symbol.
The best candidate that passes the CRC wins. Only the n_check best distinct candidates of each pass are checked, to keep the false-decode rate low. Quality figure: z = (score - sum L) / sqrt(sum V).
GF = galois.GF(p.M)
rs = galois.ReedSolomon(p.M - 1, p.M - 1 - (p.n - p.k), field=GF) # shortened to RS(n, k) by encoding k symbols
G = rs.encode(GF.Identity(p.k)) # k x n systematic generator matrix
def first_valid(sc, C, n_check):
"""Index and text of the best of the n_check best distinct candidates that passes the message checks."""
seen = set()
for i in np.argsort(-sc):
if C[i].tobytes() in seen:
continue
seen.add(C[i].tobytes())
text = symbols_to_text(C[i, :p.k], p)
if text is not None:
return i, text
if len(seen) >= n_check:
break
return None, None
def decode(E, V, n_bases, list_depth, n_check, n_free_sets, free_depth, seed=0):
"""Returns (text, codeword, score), or (None, None, None)."""
n, k = p.n, p.k
rows = np.arange(n)
order = np.argsort(-E, axis=1)[:, :max(list_depth, free_depth, 2)] # strongest tones per position
rel = (E[rows, order[:, 0]] - E[rows, order[:, 1]]) / np.sqrt(V) # reliability per position
by_rel = np.argsort(-rel)
# pass 1: k positions from the lists
weight = np.empty(n)
weight[by_rel] = np.exp(-np.arange(n) / k)
weight /= weight.sum()
rng = np.random.default_rng(seed)
combos = np.array(list(itertools.product(range(list_depth), repeat=k)))
tried, scores, cands = set(), [], []
for b in range(n_bases):
S = np.sort(by_rel[:k] if b == 0 else rng.choice(n, size=k, replace=False, p=weight))
if tuple(S) in tried:
continue
tried.add(tuple(S))
P = np.linalg.inv(G[:, S]) @ G # symbols at S -> codeword
C = np.asarray(GF(order[S[None, :], combos]) @ P, dtype=np.int64)
sc = E[rows, C].sum(axis=1)
keep = np.argsort(-sc)[:n_check]
scores.append(sc[keep])
cands.append(C[keep])
sc, C = np.concatenate(scores), np.concatenate(cands)
i, text = first_valid(sc, C, n_check)
if text is not None:
return text, C[i], sc[i]
# pass 2: k-1 positions from the lists, the remaining information symbol u over all M values. The codewords
# through k-1 fixed symbols form a line c = base + u * P[-1] over GF(M) (addition = XOR).
sets = sorted(itertools.combinations(range(n), k - 1), key=lambda S: -rel[list(S)].sum())[:n_free_sets]
combos = np.array(list(itertools.product(range(free_depth), repeat=k - 1)))
u = GF(np.arange(p.M))
Ef = E.astype(np.float32)
cands = []
for S in sets:
S = np.array(S)
f, *others = np.setdiff1d(rows, S) # free position, the rest
P = np.linalg.inv(G[:, np.r_[S, f]]) @ G
base = np.asarray(GF(order[S[None, :], combos]) @ P[:-1], dtype=np.int64) # (combos, n), 0 at f
line = np.asarray(u[:, None] * P[-1][None, :], dtype=np.int64) # (M, n), u at f
sc = Ef[S, base[:, S]].sum(axis=1)[:, None] + Ef[f][None, :] # (combos, M)
for j in others:
sc += Ef[j][base[:, j, None] ^ line[None, :, j]]
ci, ui = np.unravel_index(np.argpartition(-sc, n_check, axis=None)[:n_check], sc.shape)
cands.append(base[ci] ^ line[ui])
C = np.concatenate(cands)
sc = E[rows, C].sum(axis=1)
i, text = first_valid(sc, C, n_check)
return (text, C[i], sc[i]) if text is not None else (None, None, None)
Accumulate and decode¶
E = np.zeros((p.n, p.M)) # energy per code symbol and tone value, summed over all dwells and recordings
L = np.zeros(p.n) # its noise mean per code symbol
V = np.zeros(p.n) # its noise variance per code symbol
for i, r in enumerate(recordings):
rec = Recording(r["metas"], pol_phase_deg)
assert rec.fs == p.fs, "sample rate differs from the tx file"
if r["receiver"] is None: # simulated echo: no Doppler, echo at tx time
doppler = None
t_echo = (np.datetime64(round(r["tx_unix"] * 1e9), "ns") - rec.start) / np.timedelta64(1, "s")
else:
doppler = RadarDoppler(rec, r["target"], tx_site, r["receiver"])
t_echo = doppler.echo_start(r["tx_unix"])
D, n = dwell_energies(rec, doppler, int(round(t_echo * rec.fs)))
flag = flag_rfi(D, n, rfi_dwells, rfi_threshold)
D = np.where(flag, n[:, None], D) # flagged cells -> noise mean
pn = scramble_pn(p, r["tx_index"])
D = D[np.arange(p.n_dwells)[:, None], np.arange(p.M)[None, :] ^ pn[:, None]] # column v = symbol value v
symbol = np.arange(p.n_dwells) % p.n
np.add.at(E, symbol, D)
np.add.at(L, symbol, n)
np.add.at(V, symbol, n * var_factor)
print("%2d %s tx%02d echo %s %s %d/%d dwells RFI %.2f%%"
% (i, r["name"], r["tx_index"], str(rec.start + np.timedelta64(round(t_echo * 1e9), "ns"))[:23],
"no Doppler (sim)" if doppler is None else "Doppler %.1f Hz" % np.interp(t_echo, doppler.t, doppler.doppler),
(n > 0).sum(), p.n_dwells, 100 * flag[n > 0].mean()))
text, codeword, score = decode(E, V, **osd)
if text is None:
print(" no decode")
else:
print(" '%s' z = %.1f" % (text, (score - L.sum()) / np.sqrt(V.sum())))
print("\nDECODED: '%s'" % text if text is not None else "\nNOT DECODED")
0 wsrt_venus_2026_10_18_11_00_45_1299.650MHz_0.25Msps_ci16_le.chan0+1.sigmf-meta tx04 echo 2026-10-18T11:05:24.560 Doppler 28985.1 Hz 101/101 dwells RFI 0.36%
no decode
1 wsrt_venus_2026_10_18_11_50_54_1299.650MHz_0.25Msps_ci16_le.chan0+1.sigmf-meta tx09 echo 2026-10-18T11:55:33.494 Doppler 28338.4 Hz 101/101 dwells RFI 0.37%
'VENUS ECHO' z = 11.9
DECODED: 'VENUS ECHO'