From a473d1bb8d3ae709bbf09e41001de434d69c0202 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 21 Jul 2026 10:16:09 -0400 Subject: [PATCH 1/2] sigil files --- #module.sigil | 42 +++++++++++++++++++ .sigil/config.json | 20 +++++++++ src/ria_toolkit_oss/agent/agent.sigil | 25 +++++++++++ .../annotations/annotations.sigil | 27 ++++++++++++ src/ria_toolkit_oss/app/app.sigil | 24 +++++++++++ src/ria_toolkit_oss/data/data.sigil | 34 +++++++++++++++ src/ria_toolkit_oss/io/io.sigil | 28 +++++++++++++ .../orchestration/orchestration.sigil | 26 ++++++++++++ .../remote_control/remote-control.sigil | 25 +++++++++++ src/ria_toolkit_oss/sdr/sdr.sigil | 29 +++++++++++++ src/ria_toolkit_oss/server/server.sigil | 28 +++++++++++++ src/ria_toolkit_oss/signal/signal.sigil | 26 ++++++++++++ .../transforms/transforms.sigil | 21 ++++++++++ src/ria_toolkit_oss_cli/cli.sigil | 11 +++++ 14 files changed, 366 insertions(+) create mode 100644 #module.sigil create mode 100644 .sigil/config.json create mode 100644 src/ria_toolkit_oss/agent/agent.sigil create mode 100644 src/ria_toolkit_oss/annotations/annotations.sigil create mode 100644 src/ria_toolkit_oss/app/app.sigil create mode 100644 src/ria_toolkit_oss/data/data.sigil create mode 100644 src/ria_toolkit_oss/io/io.sigil create mode 100644 src/ria_toolkit_oss/orchestration/orchestration.sigil create mode 100644 src/ria_toolkit_oss/remote_control/remote-control.sigil create mode 100644 src/ria_toolkit_oss/sdr/sdr.sigil create mode 100644 src/ria_toolkit_oss/server/server.sigil create mode 100644 src/ria_toolkit_oss/signal/signal.sigil create mode 100644 src/ria_toolkit_oss/transforms/transforms.sigil create mode 100644 src/ria_toolkit_oss_cli/cli.sigil diff --git a/#module.sigil b/#module.sigil new file mode 100644 index 0000000..4d7aa7a --- /dev/null +++ b/#module.sigil @@ -0,0 +1,42 @@ +component RiaToolkitOss { + goal { + RIA Toolkit OSS is the open-source RIA Toolkit: the foundational Python tools to develop, test, and deploy radio intelligence applications. + It owns the core radio data types, signal generation and processing, the radio dataset framework, a unified SDR hardware abstraction, annotation tooling, visualization, and the CLI, agent, and server surfaces that expose them. + It serves both standalone SDR/ML researchers and RIA Hub, whose controller embeds this library in-process. + It does not own RIA Hub's web platform, storage, or orchestration UI; it provides the radio primitives those build on. + } + + interface { + A Python library (ria_toolkit_oss) importable by downstream code, including RIA Hub's controller which imports it in-process. + Library surfaces: data (radio-ML data types including Recording and Annotation, plus the radio dataset and dataset-builder framework); io (load/save recordings across SigMF, npy, wav, and Blue); signal (signal generation and processing); transforms (NumPy radio-data transforms used in ML); sdr (unified receive/transmit API across software-defined radios); annotations (annotation management and automatic signal-detection annotation); view and viz (plotting and report generation); utils (array-conversion and helpers). + orchestration: configure and run automated RF capture campaigns (CampaignConfig). + remote_control: remote SDR transmitter control over SSH and ZMQ. + server: the RT-OSS HTTP server (ria-server) that can run orchestration and inference as a standalone HTTP service. + CLI suite (console scripts): ria and ria-tools (main toolkit CLI), ria-server (RT-OSS HTTP server), ria-agent (edge node agent for campaigns/inference), ria-app (pull and run containerized RIA applications). + File formats: SigMF signal recordings and HDF5 radio datasets. + } +} + +expand RiaToolkitOss { + logic { + The library provides the radio data types and signal tools; the CLI, agent, server, and app runners are entry points layered over that library. + It is consumed both as an imported Python library and through its console-script CLIs. + The server package (RT-OSS) is a standalone HTTP deployment of orchestration and inference; RIA Hub's controller embeds this library in-process rather than calling that server. + } + + constraints { + Python 3.10+; license is AGPL-3.0-only. + Ships as a single distribution (ria-toolkit-oss) containing two packages from src: the ria_toolkit_oss library and the ria_toolkit_oss_cli CLI. + numpy is pinned to 1.26.4 for Radioconda compatibility; scipy is constrained below 1.16. + SDR hardware backends are intended as optional extras, but the individual optional-dependency entries are currently commented out; only an all-sdr list (pyrtlsdr, pyadi-iio for Pluto, pyhackrf, pyrf for ThinkRF; excluding USRP/UHD and BladeRF, which need system libraries) is present, and it is not wired as a valid PEP 621 extra. + Built with poetry-core. + The view and viz packages overlap: both provide recording-plot primitives (spectrogram, iq_time_series, frequency_spectrum, constellation); viz is the superset RIA Hub imports, view is the CLI-facing set. This duplication is unresolved. + } + + cases { + A researcher installs the package and uses the ria CLI to generate or inspect a radio dataset. + RIA Hub's controller imports ria_toolkit_oss in-process to load recordings, render plots, and drive radios. + A ria-agent node runs on an edge device to execute capture or inference campaigns. + ria-server exposes toolkit operations to RIA Hub over HTTP. + } +} diff --git a/.sigil/config.json b/.sigil/config.json new file mode 100644 index 0000000..8348c1a --- /dev/null +++ b/.sigil/config.json @@ -0,0 +1,20 @@ +{ + "sigilVersion": "0.1.0", + "workspace": { + "name": "ria-toolkit-oss", + "members": [] + }, + "files": { + "include": [ + "**/*.sigil" + ], + "exclude": [ + ".git/**", + ".deno/**", + "node_modules/**", + "build/**", + "coverage/**" + ] + }, + "tools": {} +} diff --git a/src/ria_toolkit_oss/agent/agent.sigil b/src/ria_toolkit_oss/agent/agent.sigil new file mode 100644 index 0000000..2a3ab28 --- /dev/null +++ b/src/ria_toolkit_oss/agent/agent.sigil @@ -0,0 +1,25 @@ +component RiaAgent { + goal { + Run a toolkit agent on an edge device that connects outbound to RIA Hub to capture, stream, or run inference on live radio. + Own the agent CLI, its two execution modes, and its persisted registration. + } + + interface { + CLI ria-agent with subcommands: register (--hub, --api-key, --name, and TX interlock flags), stream, run (legacy), and detect (list SDR drivers). + register POSTs to {hub}/screens/agents/register with the X-API-Key registration key and saves agent_id and token to ~/.ria/agent.json (mode 0600). + Two execution modes: a WebSocket streamer that opens an SDR and streams raw IQ to the hub (the hub does inference), and a legacy long-poll NodeAgent that runs ONNX inference locally on the device. + } +} + +expand RiaAgent { + logic { + The streamer connects to wss://{hub}/screens/agent/ws with a bearer token, heartbeats, auto-reconnects, and runs concurrent RX (ship interleaved float32 IQ) and TX (play inbound IQ) sessions, sharing one driver for full-duplex when RX and TX name the same device. + The legacy NodeAgent connects outbound only: it registers at /composer/nodes/register, heartbeats, long-polls /commands, and dispatches run_campaign, load_model, start/stop/configure_inference, and TX commands, reporting to /events and /campaign-status. + } + + constraints { + TX is opt-in (--allow-tx) and bounded by tx_max_gain_db, tx_max_duration_s, and allowed frequency ranges, with a TX watchdog; a TX-start request exceeding the interlocks is rejected (per-frame handling rejects malformed frames and enforces max duration). + Registration keys are personal (ria_reg_ prefix); the legacy shared key is deprecated. Failures are reason-coded (invalid_key, expired, revoked, already_consumed, rate-limited). + Config is AgentConfig at ~/.ria/agent.json (override RIA_AGENT_CONFIG). + } +} diff --git a/src/ria_toolkit_oss/annotations/annotations.sigil b/src/ria_toolkit_oss/annotations/annotations.sigil new file mode 100644 index 0000000..944aa3a --- /dev/null +++ b/src/ria_toolkit_oss/annotations/annotations.sigil @@ -0,0 +1,27 @@ +component Annotations { + goal { + Create, manage, and automatically detect annotations on a recording. + Own the detection algorithms, annotation transforms, signal isolation, and slice qualification. + } + + interface { + Automatic detection, each returning a new Recording with detected annotations appended: detect_signals_energy (energy/envelope), annotate_with_cusum (change-point segmentation), threshold_qualifier (hysteresis burst detection), annotate_with_obw (occupied-bandwidth), and find_spectral_components with split_recording_annotations (parallel signal separation). + Bandwidth helpers: calculate_occupied_bandwidth, calculate_nominal_bandwidth, calculate_full_detected_bandwidth. + Annotation transforms: remove_contained_boxes and is_annotation_contained (merge_annotations is declared but not implemented). + isolate_signal (slice, normalize, shift to baseband, low-pass to the annotation bandwidth) and qualify_slice_from_annotations (fixed-length slices overlapping annotations). + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } +} + +expand Annotations { + logic { + Energy detection builds a moving-average envelope, estimates a segment-median noise floor, thresholds with hysteresis, merges nearby boundaries, and assigns frequency bounds by the chosen method (nominal, occupied, or full-detected bandwidth). + Detectors write a JSON comment recording the detection type, generator, and parameters, plus a detail dict. + } + + constraints { + Detectors require a sample_rate in the recording metadata and operate on channel 0 (only threshold_qualifier accepts a channel override). + Annotation frequencies are stored absolute (center frequency plus the relative offset); the parallel separator works in baseband and converts. + A detector warns and returns the recording unchanged when the dynamic range is too low to threshold. + } +} diff --git a/src/ria_toolkit_oss/app/app.sigil b/src/ria_toolkit_oss/app/app.sigil new file mode 100644 index 0000000..d1917e3 --- /dev/null +++ b/src/ria_toolkit_oss/app/app.sigil @@ -0,0 +1,24 @@ +component AppRunner { + goal { + Pull and run containerized RIA applications, auto-configuring hardware access from image labels. + Own the ria-app CLI over a container engine. + } + + interface { + CLI ria-app with subcommands: pull, run, list, stop, logs, and configure; a global --sudo flag. + run auto-pulls the image if absent and derives container flags from the image's ria.* labels. + } +} + +expand AppRunner { + logic { + The engine is the first of docker or podman on PATH, optionally sudo-prefixed. + Image references resolve against a configured registry and namespace; fully-qualified references pass through. + Hardware flags are derived from ria.profile and ria.hardware labels: GPU access when an NVIDIA profile and runtime are present, USB device passthrough for USB SDRs, and host networking for USRP, ThinkRF, and Pluto; flags can be overridden or dry-run. + } + + constraints { + There is no Python SDK; it is a subprocess wrapper and exits with an error if no container engine is found. + Config is AppConfig (registry, namespace, sudo) at ~/.ria/toolkit.json with environment-variable fallbacks. + } +} diff --git a/src/ria_toolkit_oss/data/data.sigil b/src/ria_toolkit_oss/data/data.sigil new file mode 100644 index 0000000..7018659 --- /dev/null +++ b/src/ria_toolkit_oss/data/data.sigil @@ -0,0 +1,34 @@ +component RadioData { + goal { + Provide the core radio-ML data types: a Recording (an immutable IQ tape with metadata and annotations) and the radio dataset and dataset-builder framework. + Own the in-memory contract the rest of the toolkit and RIA Hub build on for signals and datasets. + } + + interface { + Recording(data, metadata=None, dtype=None, timestamp=None, annotations=None) - complex IQ data shaped channels-by-samples (1-D is reshaped to one channel); metadata must be a JSON-serializable dict; rec_id and timestamp are auto-populated. + Recording properties: data, metadata, annotations, shape, n_chan, rec_id, dtype, timestamp, sample_rate (settable). + Recording methods: astype, add_to_metadata/update_metadata/remove_from_metadata (rec_id and timestamp protected), trim, normalize, view/simple_view, and to_sigmf/to_npy/to_wav/to_blue (delegating to io). + Annotation(sample_start, sample_count, freq_lower_edge=None, freq_upper_edge=None, label="", comment="", detail=None) with is_valid, overlap, area, and to_sigmf_format. + Dataset framework (data.datasets): abstract RadioDataset (HDF5-backed and iterable; item assignment raises, though it exposes explicit inplace mutators) with IQDataset and SpectDataset subclasses, DatasetBuilder, and leakage-safe split/random_split. + } +} + +expand RadioData { + state { + A Recording's sample data is immutable: item assignment raises, and property accessors return copies or read-only views; metadata is mutable only through the metadata methods. + Metadata keys must be lowercase letters and underscores only (no digits); rec_id and timestamp are protected and cannot be modified or removed. + } + + logic { + A Recording requires complex data of one or two dimensions (channels by samples); rec_id is the SHA-256 of the data bytes plus timestamp. + trim slices samples and re-aligns annotations; normalize scales peak magnitude to 1 and rejects all-zero data. + RadioDataset reads data and metadata lazily from an HDF5 source; non-inplace mutations write numbered sibling files; concrete backends live outside the OSS package. + split and random_split partition by rec_id so no recording leaks across subsets; lengths may be counts or fractions. + } + + constraints { + Recording data must be complex or construction raises; metadata must be a JSON-serializable dict. + RadioDataset sources must be HDF5 (validated at construction); the data and metadata datasets are required for use but not checked at init; the class is abstract and instantiated only via a concrete backend. + The datatypes package is stale (only bytecode remains); the live location is data. + } +} diff --git a/src/ria_toolkit_oss/io/io.sigil b/src/ria_toolkit_oss/io/io.sigil new file mode 100644 index 0000000..c29cdb0 --- /dev/null +++ b/src/ria_toolkit_oss/io/io.sigil @@ -0,0 +1,28 @@ +component RecordingIO { + goal { + Load and save Recording objects to and from file across the RF formats the toolkit supports. + Own format detection and the per-format readers and writers. + } + + interface { + load_recording(file) - dispatch on file extension to the matching reader (sigmf, npy, wav, blue); an unknown extension raises. + Per-format readers and writers: to_sigmf/from_sigmf, to_npy/from_npy (plus from_npy_legacy), to_wav/from_wav, to_blue/from_blue. + Most writers return the output path (to_sigmf returns None) and honor an overwrite flag; auto-generated filenames land under recordings/. + } +} + +expand RecordingIO { + logic { + The npy v2 format stores a version marker then data, JSON metadata, and JSON annotations, avoiding pickle; legacy npy paths fall back to pickle with a warning and normalize namespaced metadata keys. + SigMF writes paired .sigmf-data and .sigmf-meta, mapping RIA metadata to core keys and namespacing the rest under ria:. + WAV stores I and Q as stereo channels with metadata in an info chunk; Blue uses the MIDAS 512-byte header format. + } + + constraints { + Format detection is by file extension only. + The SigMF, WAV, and Blue writers reject multichannel input and the readers produce single-channel output; only npy round-trips multichannel data. + Integer-encoded outputs (16-bit WAV, integer Blue) require samples normalized to the range -1 to 1. + io.common (exists, validate, move, copy) is declared but unimplemented. + save_recording is exported in the package __all__ but not defined (latent bug). + } +} diff --git a/src/ria_toolkit_oss/orchestration/orchestration.sigil b/src/ria_toolkit_oss/orchestration/orchestration.sigil new file mode 100644 index 0000000..d9491a8 --- /dev/null +++ b/src/ria_toolkit_oss/orchestration/orchestration.sigil @@ -0,0 +1,26 @@ +component Orchestration { + goal { + Define and execute automated RF capture campaigns: sweep transmitters and channels, record, QA, and label the results. + Own the campaign configuration schema, the executor, and the QA checks. + } + + interface { + CampaignConfig(name, recorder, transmitters, qa, output, mode, loops) with loaders from_dict, from_yaml, and from_device_profile; it requires a recorder and at least one transmitter. + Supporting dataclasses: RecorderConfig, TransmitterConfig (type and control_method external_script/sdr/sdr_remote/sdr_agent), CaptureStep, QAConfig, and OutputConfig. + CampaignExecutor(config, progress_cb=None).run() returns a CampaignResult of StepResults; check_recording(recording, config) returns a QAResult; label_recording writes ria: SigMF metadata. + } +} + +expand Orchestration { + logic { + The executor opens the SDR and remote TX controllers once, then for each loop, transmitter, and step it starts the transmitter, records IQ, stops the transmitter, then labels, QA-checks, and saves it as SigMF. + QA estimates SNR from the PSD and checks duration and SNR; in flag-for-review mode a recording always passes but may be flagged. + Cancellation is cooperative at step boundaries via the progress callback. + } + + constraints { + Frequencies, gains, durations, and bandwidths are parsed from human units (for example 2.45GHz, auto, 1.5m). + Device names are aliased onto SDR drivers, including mock and sim for hardware-free runs. + The controller's Conductor and the RT-OSS server are parallel HTTP wrappers that both reuse this package's CampaignConfig and CampaignExecutor in-process. + } +} diff --git a/src/ria_toolkit_oss/remote_control/remote-control.sigil b/src/ria_toolkit_oss/remote_control/remote-control.sigil new file mode 100644 index 0000000..5e93fbb --- /dev/null +++ b/src/ria_toolkit_oss/remote_control/remote-control.sigil @@ -0,0 +1,25 @@ +component RemoteControl { + goal { + Drive a software-defined radio transmitter on a remote machine over SSH and ZMQ. + Own the client controller and the remote server that bridge transmit commands to a remote SDR. + } + + interface { + RemoteTransmitterController(host, ssh_user, ssh_key_path, zmq_port=5556): set_radio, init_tx, transmit_async, wait_transmit, stop, close. + RemoteTransmitter (the server on the transmit machine): set_radio, init_tx, transmit, stop, and a run_function JSON-RPC dispatcher; launched as a module over a ZMQ REP socket. + } +} + +expand RemoteControl { + logic { + The controller SSHes into the transmit host (paramiko, key-file auth), launches the remote_transmitter server there, waits for it to bind, then drives it with JSON-RPC over a ZMQ REQ/REP socket. + Supported remote devices are the transmit-capable drivers: Pluto, USRP, HackRF, and BladeRF. + set_radio must precede init_tx and transmit (both guard on a radio being set); the init_tx-before-transmit order is a usage convention, not an enforced guard; transmit runs in a daemon thread joined by wait_transmit. + } + + constraints { + SSH host-key policy is AutoAddPolicy, which accepts unknown hosts (security note). + RIA Hub does not use this subpackage; remote and agent transmit is handled through the agent path instead. + Known gap: transmit calls an SDR method tx_cw that no driver defines, so remote CW transmit currently no-ops for real drivers (the AttributeError is swallowed). + } +} diff --git a/src/ria_toolkit_oss/sdr/sdr.sigil b/src/ria_toolkit_oss/sdr/sdr.sigil new file mode 100644 index 0000000..7ca0893 --- /dev/null +++ b/src/ria_toolkit_oss/sdr/sdr.sigil @@ -0,0 +1,29 @@ +component SdrInterface { + goal { + Provide a unified receive and transmit API across a variety of software-defined radios. + Own the SDR base contract, the per-device drivers, and device discovery and selection. + } + + interface { + get_sdr_device(device_type, ident=None, tx=False) returns an SDR; mock and sim return a MockSDR, real devices delegate to the CLI package's factory. + detect_available() returns the drivers whose optional dependency imports cleanly; it does not probe hardware. + SDR base class: record(num_samples, rx_time) and rx(num_samples) to receive, tx_recording(recording) to transmit, stream_to_zmq and pickle_buffer_to_zmq to stream, getters and setters for sample rate, center frequency, and gain, and capability probes for bias-tee and dynamic-update support. + Drivers: Pluto, USRP, HackRF, and Blade (receive and transmit); RTLSDR and ThinkRF (receive only); MockSDR (AWGN generator, no hardware, receive and transmit). Real drivers are constructed with a device identifier; MockSDR takes a buffer size and seed. + Errors: SDRError with SDRParameterError and SdrDisconnectedError (SDROverflowError is defined but not in the public __all__). + } +} + +expand SdrInterface { + logic { + init_rx or init_tx must be called before record or tx_recording (rx auto-initializes on first use); they push sample rate, center frequency, and gain to the hardware via per-driver setters. + Receive accumulates validated buffers into a Recording; transmit loops or truncates the source IQ to the requested length. + A driver is available only if its optional hardware dependency imports; USB or network drops are translated to SdrDisconnectedError. + } + + constraints { + gain_mode absolute applies the requested gain (clamped or snapped to the device's valid range); relative requires a negative (attenuation) value; ThinkRF uses named gain profiles instead. + RTLSDR and ThinkRF are receive-only; their transmit methods raise or are stubs. + Identifiers are device-specific free-form values (Pluto URI or IP, USRP serial or args, HackRF/BladeRF serial). + RIA Hub consumes this layer through the SDR base contract and get_sdr_device but currently wires only Pluto and USRP live; the controller imports driver modules directly rather than only via the factory. + } +} diff --git a/src/ria_toolkit_oss/server/server.sigil b/src/ria_toolkit_oss/server/server.sigil new file mode 100644 index 0000000..04a1f89 --- /dev/null +++ b/src/ria_toolkit_oss/server/server.sigil @@ -0,0 +1,28 @@ +component RtOssServer { + goal { + Expose the toolkit's campaign orchestration and live RF inference over HTTP as a standalone service (RT-OSS). + Own the FastAPI app, its auth, and the deploy and inference endpoints. + } + + interface { + create_app(api_key="") returns a FastAPI app mounting /conductor and /inference (both API-key gated) plus an unauthenticated GET /health; an empty api_key disables auth (dev only). + serve() launches uvicorn from RT_OSS_API_KEY, RT_OSS_HOST (default 0.0.0.0), and RT_OSS_PORT (default 8080), and is the ria-server entry point; the ria serve CLI command is separate and calls create_app() directly. + Conductor endpoints: POST /conductor/deploy ({config}) returns {campaign_id} and runs the campaign in a background thread; GET /conductor/status/{campaign_id}; POST /conductor/cancel/{campaign_id}. + Inference endpoints: POST /inference/load (ONNX model plus label_map), POST /inference/start, POST /inference/stop, POST /inference/configure, GET /inference/status. + } +} + +expand RtOssServer { + logic { + A deploy parses the body via CampaignConfig.from_dict and rejects any transmitter carrying a script field (external scripts blocked). + The inference loop captures 4096 IQ samples, estimates SNR, preprocesses to the model input shape, softmaxes, and maps the index to a label; idle-class labels are reported as idle. + State is in-memory only (a campaign dict plus a single thread-locked inference state); nothing is persisted. + } + + constraints { + Auth is X-API-Key compared in constant time; a missing configured key allows all requests (dev only). + Server dependencies (fastapi, uvicorn, onnxruntime) are an optional extra; a missing onnxruntime or SDR yields 503. + RIA Hub's controller does not call this server; it embeds the toolkit library in-process. This server is the alternative standalone deployment. + model_path is path-traversal validated and must resolve to a .onnx file. + } +} diff --git a/src/ria_toolkit_oss/signal/signal.sigil b/src/ria_toolkit_oss/signal/signal.sigil new file mode 100644 index 0000000..7bf160e --- /dev/null +++ b/src/ria_toolkit_oss/signal/signal.sigil @@ -0,0 +1,26 @@ +component SignalProcessing { + goal { + Provide the toolkit's signal generation and processing suite: functional waveform generators and a modular block-based modulation-chain framework. + Own the DSP building blocks that produce Recording objects for datasets and transmission. + } + + interface { + Functional generators returning a Recording: sine, square, sawtooth, noise, chirp, lfm_chirp_complex, complex_sine. + Block framework: Block, SourceBlock, ProcessBlock, and RecordableBlock forming a DataType-typed chain, with sources (BinarySource, AWGNSource, SineSource, RecordingSource, ...), Mapper (PSK/QAM/PAM constellations; standalone APSK and cross-QAM mapper classes exist but are not reachable through Mapper) and SymbolDemapper, symbol modulators (GMSK, OOK, OQPSK), continuous-phase modulators (FSK, CPFSK), pulse-shaping filters (RaisedCosine, RootRaisedCosine, Gaussian, Sinc, Rect), Upsampling, channels (AWGNChannel, FlatRayleigh), and basic ops (Add, MultiplyConstant, PhaseShift, FrequencyShift, frequency up/down conversion). + High-level generators: PSKGenerator, QAMGenerator, and PAMGenerator, each with record(batch_size, num_bits). + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } +} + +expand SignalProcessing { + logic { + A SignalGenerator validates that each block's output type matches the next block's input type; sample production is driven by the blocks themselves (Block and RecordableBlock via get_samples, and the PSK/QAM/PAM generators by chaining block __call__). + Blocks auto-serialize their JSON-safe attributes into the produced Recording's metadata. + } + + constraints { + Supported modulations reachable via Mapper and the generators: PSK, QAM, PAM (constellation); GMSK, OOK, OQPSK (symbol); FSK and CPFSK (continuous). QAM and PAM require an even bits-per-symbol (QAM additionally more than 2). APSK and cross-QAM mapper classes exist but are unwired. + The basic generators (sine, square, sawtooth, noise, complex_sine) raise on a sample rate below 1 (chirp instead requires at least two samples); FSK and CPFSK require frequency spacing times symbol duration of at least 0.5. + Data flows as typed stages (bits, symbols, upsampled symbols, baseband, passband) via the DataType enum. + } +} diff --git a/src/ria_toolkit_oss/transforms/transforms.sigil b/src/ria_toolkit_oss/transforms/transforms.sigil new file mode 100644 index 0000000..3714156 --- /dev/null +++ b/src/ria_toolkit_oss/transforms/transforms.sigil @@ -0,0 +1,21 @@ +component Transforms { + goal { + Provide NumPy radio-data transforms used by the ML backends: data augmentations and channel/hardware impairment models over IQ. + Own the functions that manipulate a signal while preserving its container type. + } + + interface { + Augmentations (iq_augmentations): generate_awgn, time_reversal, spectral_inversion, channel_swap, amplitude_reversal, drop_samples, quantize_tape, quantize_parts, magnitude_rescale, cut_out, patch_shuffle. + Impairments (iq_impairments): add_awgn_to_signal, time_shift, frequency_shift, phase_shift, iq_imbalance, resample. + Every function accepts an array or a Recording and returns the same type, preserving metadata for a Recording. + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } +} + +expand Transforms { + constraints { + Input must be a 2-D complex channels-by-samples array or a ValueError is raised; most transforms implement only the single-channel path and raise NotImplementedError for more channels. + frequency_shift is relative to the sample rate and must be within -0.5 to 0.5; phase_shift must be within -pi to pi. + AWGN is generated to match a target SNR in dB. + } +} diff --git a/src/ria_toolkit_oss_cli/cli.sigil b/src/ria_toolkit_oss_cli/cli.sigil new file mode 100644 index 0000000..e7de867 --- /dev/null +++ b/src/ria_toolkit_oss_cli/cli.sigil @@ -0,0 +1,11 @@ +component ToolkitCli { + goal { + Provide the ria and ria-tools command-line interface over the toolkit's library features. + Own the Click command group and its subcommands. + } + + interface { + A Click group that auto-registers the commands defined in ria_toolkit_oss_cli.ria_toolkit_oss.commands and warns when git-lfs is missing. + Commands map to library features: capture and transmit (sdr), campaign (orchestration), generate and synth (signal), transform/combine/split/convert (transforms, data, io), annotate (annotations), view (view and viz), discover (sdr), serve (the RT-OSS server), and upload/setup_repo/init (RIA Hub repo integration). + } +} -- 2.34.1 From 1a8bfb2c5153f9b79f48565c11e1e08b40e1e43e Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 28 Jul 2026 10:27:27 -0400 Subject: [PATCH 2/2] sigil 0.5 update --- #module.sigil | 67 +++++++++++++++++-- .sigil/config.json | 2 +- src/ria_toolkit_oss/agent/agent.sigil | 40 ++++++++++- .../annotations/annotations.sigil | 48 +++++++++++-- src/ria_toolkit_oss/app/app.sigil | 25 ++++++- src/ria_toolkit_oss/data/data.sigil | 59 ++++++++++++++-- src/ria_toolkit_oss/io/io.sigil | 33 ++++++++- .../orchestration/orchestration.sigil | 29 +++++++- .../remote_control/remote-control.sigil | 26 ++++++- src/ria_toolkit_oss/sdr/sdr.sigil | 28 ++++++-- src/ria_toolkit_oss/server/server.sigil | 43 ++++++++++-- src/ria_toolkit_oss/signal/signal.sigil | 45 +++++++++++-- .../transforms/transforms.sigil | 34 ++++++++-- src/ria_toolkit_oss_cli/cli.sigil | 10 ++- 14 files changed, 439 insertions(+), 50 deletions(-) diff --git a/#module.sigil b/#module.sigil index 4d7aa7a..8c0ac9c 100644 --- a/#module.sigil +++ b/#module.sigil @@ -1,42 +1,95 @@ component RiaToolkitOss { goal { RIA Toolkit OSS is the open-source RIA Toolkit: the foundational Python tools to develop, test, and deploy radio intelligence applications. + It owns the core radio data types, signal generation and processing, the radio dataset framework, a unified SDR hardware abstraction, annotation tooling, visualization, and the CLI, agent, and server surfaces that expose them. + It serves both standalone SDR/ML researchers and RIA Hub, whose controller embeds this library in-process. + It does not own RIA Hub's web platform, storage, or orchestration UI; it provides the radio primitives those build on. } interface { - A Python library (ria_toolkit_oss) importable by downstream code, including RIA Hub's controller which imports it in-process. - Library surfaces: data (radio-ML data types including Recording and Annotation, plus the radio dataset and dataset-builder framework); io (load/save recordings across SigMF, npy, wav, and Blue); signal (signal generation and processing); transforms (NumPy radio-data transforms used in ML); sdr (unified receive/transmit API across software-defined radios); annotations (annotation management and automatic signal-detection annotation); view and viz (plotting and report generation); utils (array-conversion and helpers). - orchestration: configure and run automated RF capture campaigns (CampaignConfig). - remote_control: remote SDR transmitter control over SSH and ZMQ. - server: the RT-OSS HTTP server (ria-server) that can run orchestration and inference as a standalone HTTP service. - CLI suite (console scripts): ria and ria-tools (main toolkit CLI), ria-server (RT-OSS HTTP server), ria-agent (edge node agent for campaigns/inference), ria-app (pull and run containerized RIA applications). - File formats: SigMF signal recordings and HDF5 radio datasets. + PythonLibrary { + A Python library (ria_toolkit_oss) importable by downstream code, including RIA Hub's controller which imports it in-process. + + Library surfaces: data (radio-ML data types including Recording and Annotation, plus the radio dataset and dataset-builder framework); io (load/save recordings across SigMF, npy, wav, and Blue); signal (signal generation and processing); transforms (NumPy radio-data transforms used in ML); sdr (unified receive/transmit API across software-defined radios); annotations (annotation management and automatic signal-detection annotation); view and viz (plotting and report generation); utils (array-conversion and helpers). + } + + Orchestration { + orchestration: configure and run automated RF capture campaigns (CampaignConfig). + } + + RemoteControl { + remote_control: remote SDR transmitter control over SSH and ZMQ. + } + + Server { + server: the RT-OSS HTTP server (ria-server) that can run orchestration and inference as a standalone HTTP service. + } + + CommandLineTools { + CLI suite (console scripts): ria and ria-tools (main toolkit CLI), ria-server (RT-OSS HTTP server), ria-agent (edge node agent for campaigns/inference), ria-app (pull and run containerized RIA applications). + } + + FileFormats { + File formats: SigMF signal recordings and HDF5 radio datasets. + } } } expand RiaToolkitOss { logic { The library provides the radio data types and signal tools; the CLI, agent, server, and app runners are entry points layered over that library. + It is consumed both as an imported Python library and through its console-script CLIs. + The server package (RT-OSS) is a standalone HTTP deployment of orchestration and inference; RIA Hub's controller embeds this library in-process rather than calling that server. } constraints { Python 3.10+; license is AGPL-3.0-only. + Ships as a single distribution (ria-toolkit-oss) containing two packages from src: the ria_toolkit_oss library and the ria_toolkit_oss_cli CLI. + numpy is pinned to 1.26.4 for Radioconda compatibility; scipy is constrained below 1.16. + SDR hardware backends are intended as optional extras, but the individual optional-dependency entries are currently commented out; only an all-sdr list (pyrtlsdr, pyadi-iio for Pluto, pyhackrf, pyrf for ThinkRF; excluding USRP/UHD and BladeRF, which need system libraries) is present, and it is not wired as a valid PEP 621 extra. + Built with poetry-core. + The view and viz packages overlap: both provide recording-plot primitives (spectrogram, iq_time_series, frequency_spectrum, constellation); viz is the superset RIA Hub imports, view is the CLI-facing set. This duplication is unresolved. } + decisions { + DependencyPinning { + Decision: Pin numpy to 1.26.4 and constrain scipy below 1.16. + + Scope: Governs the distribution's core numeric dependencies; does not govern optional SDR backends. + + Design issues addressed: Radioconda compatibility requires a specific numpy ABI, and an unconstrained scipy pulls in versions incompatible with that pin. + + Revisit when: Radioconda moves to a newer numpy, or the scipy constraint blocks a required feature. + } + + SdrBackendExtras { + Decision: Offer SDR hardware backends only as optional extras, and exclude USRP/UHD and BladeRF from the bundled all-sdr list. + + Scope: Governs optional-dependency packaging for SDR hardware; the core library installs without any SDR backend. + + Design issues addressed: USRP/UHD and BladeRF require system libraries that cannot be resolved through Python packaging alone. + + Consequences: Only pyrtlsdr, pyadi-iio (Pluto), pyhackrf, and pyrf (ThinkRF) are candidates for pure Python extras; the individual entries are currently commented out and the all-sdr list is not yet wired as a valid PEP 621 extra. + } + } + cases { A researcher installs the package and uses the ria CLI to generate or inspect a radio dataset. + RIA Hub's controller imports ria_toolkit_oss in-process to load recordings, render plots, and drive radios. + A ria-agent node runs on an edge device to execute capture or inference campaigns. + ria-server exposes toolkit operations to RIA Hub over HTTP. } } diff --git a/.sigil/config.json b/.sigil/config.json index 8348c1a..0616a94 100644 --- a/.sigil/config.json +++ b/.sigil/config.json @@ -1,5 +1,5 @@ { - "sigilVersion": "0.1.0", + "sigilVersion": "0.5.0", "workspace": { "name": "ria-toolkit-oss", "members": [] diff --git a/src/ria_toolkit_oss/agent/agent.sigil b/src/ria_toolkit_oss/agent/agent.sigil index 2a3ab28..f0ee780 100644 --- a/src/ria_toolkit_oss/agent/agent.sigil +++ b/src/ria_toolkit_oss/agent/agent.sigil @@ -1,25 +1,59 @@ component RiaAgent { goal { Run a toolkit agent on an edge device that connects outbound to RIA Hub to capture, stream, or run inference on live radio. + Own the agent CLI, its two execution modes, and its persisted registration. } interface { - CLI ria-agent with subcommands: register (--hub, --api-key, --name, and TX interlock flags), stream, run (legacy), and detect (list SDR drivers). - register POSTs to {hub}/screens/agents/register with the X-API-Key registration key and saves agent_id and token to ~/.ria/agent.json (mode 0600). - Two execution modes: a WebSocket streamer that opens an SDR and streams raw IQ to the hub (the hub does inference), and a legacy long-poll NodeAgent that runs ONNX inference locally on the device. + AgentCommands { + CLI ria-agent with subcommands: register (--hub, --api-key, --name, and TX interlock flags), stream, run (legacy), and detect (list SDR drivers). + } + + Registration { + register POSTs to {hub}/screens/agents/register with the X-API-Key registration key and saves agent_id and token to ~/.ria/agent.json (mode 0600). + } + + ExecutionModes { + Two execution modes: a WebSocket streamer that opens an SDR and streams raw IQ to the hub (the hub does inference), and a legacy long-poll NodeAgent that runs ONNX inference locally on the device. + } } } expand RiaAgent { logic { The streamer connects to wss://{hub}/screens/agent/ws with a bearer token, heartbeats, auto-reconnects, and runs concurrent RX (ship interleaved float32 IQ) and TX (play inbound IQ) sessions, sharing one driver for full-duplex when RX and TX name the same device. + The legacy NodeAgent connects outbound only: it registers at /composer/nodes/register, heartbeats, long-polls /commands, and dispatches run_campaign, load_model, start/stop/configure_inference, and TX commands, reporting to /events and /campaign-status. } constraints { TX is opt-in (--allow-tx) and bounded by tx_max_gain_db, tx_max_duration_s, and allowed frequency ranges, with a TX watchdog; a TX-start request exceeding the interlocks is rejected (per-frame handling rejects malformed frames and enforces max duration). + Registration keys are personal (ria_reg_ prefix); the legacy shared key is deprecated. Failures are reason-coded (invalid_key, expired, revoked, already_consumed, rate-limited). + Config is AgentConfig at ~/.ria/agent.json (override RIA_AGENT_CONFIG). } + + decisions { + ExecutionModeSplit { + Decision: The agent offers a WebSocket streamer that ships raw IQ for hub-side inference and retains a legacy long-poll NodeAgent that runs ONNX inference locally on the device. + + Scope: Governs where inference executes for a connected agent. + + Trade-offs: Streaming keeps the edge thin and centralizes model logic at the hub at the cost of a continuous IQ uplink; local inference avoids that uplink but pins model execution to the device. + + Consequences: The legacy local-inference path is retained alongside the streamer rather than being the primary mode. + } + + TxOptIn { + Decision: Transmit is disabled unless --allow-tx is set and is bounded by max gain, max duration, allowed frequency ranges, and a TX watchdog. + + Scope: Governs whether and how a registered agent may transmit. + + Design issues addressed: RF transmission is regulated and physically risky, so an agent must not transmit by default or outside operator-declared bounds. + + Consequences: A TX-start request exceeding the interlocks is rejected. + } + } } diff --git a/src/ria_toolkit_oss/annotations/annotations.sigil b/src/ria_toolkit_oss/annotations/annotations.sigil index 944aa3a..e946bae 100644 --- a/src/ria_toolkit_oss/annotations/annotations.sigil +++ b/src/ria_toolkit_oss/annotations/annotations.sigil @@ -1,27 +1,65 @@ component Annotations { goal { Create, manage, and automatically detect annotations on a recording. + Own the detection algorithms, annotation transforms, signal isolation, and slice qualification. } interface { - Automatic detection, each returning a new Recording with detected annotations appended: detect_signals_energy (energy/envelope), annotate_with_cusum (change-point segmentation), threshold_qualifier (hysteresis burst detection), annotate_with_obw (occupied-bandwidth), and find_spectral_components with split_recording_annotations (parallel signal separation). - Bandwidth helpers: calculate_occupied_bandwidth, calculate_nominal_bandwidth, calculate_full_detected_bandwidth. - Annotation transforms: remove_contained_boxes and is_annotation_contained (merge_annotations is declared but not implemented). - isolate_signal (slice, normalize, shift to baseband, low-pass to the annotation bandwidth) and qualify_slice_from_annotations (fixed-length slices overlapping annotations). - This layer is a standalone library surface; the RIA Hub controller does not import it directly. + SignalDetection { + Automatic detection, each returning a new Recording with detected annotations appended: detect_signals_energy (energy/envelope), annotate_with_cusum (change-point segmentation), threshold_qualifier (hysteresis burst detection), annotate_with_obw (occupied-bandwidth), and find_spectral_components with split_recording_annotations (parallel signal separation). + } + + BandwidthHelpers { + Bandwidth helpers: calculate_occupied_bandwidth, calculate_nominal_bandwidth, calculate_full_detected_bandwidth. + } + + AnnotationTransforms { + Annotation transforms: remove_contained_boxes and is_annotation_contained (merge_annotations is declared but not implemented). + } + + SignalIsolation { + isolate_signal (slice, normalize, shift to baseband, low-pass to the annotation bandwidth) and qualify_slice_from_annotations (fixed-length slices overlapping annotations). + } + + StandaloneSurface { + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } } } expand Annotations { logic { Energy detection builds a moving-average envelope, estimates a segment-median noise floor, thresholds with hysteresis, merges nearby boundaries, and assigns frequency bounds by the chosen method (nominal, occupied, or full-detected bandwidth). + Detectors write a JSON comment recording the detection type, generator, and parameters, plus a detail dict. } constraints { Detectors require a sample_rate in the recording metadata and operate on channel 0 (only threshold_qualifier accepts a channel override). + Annotation frequencies are stored absolute (center frequency plus the relative offset); the parallel separator works in baseband and converts. + A detector warns and returns the recording unchanged when the dynamic range is too low to threshold. } + + decisions { + DetectionProvenance { + Decision: Have each detector write a JSON comment capturing the detection type, generator, and parameters, plus a detail dict, onto the annotations it appends. + + Scope: Governs annotations produced by the automatic detectors. + + Design issues addressed: A detected annotation's origin and settings must be recoverable after the fact for auditing and reproducibility. + } + + AbsoluteFrequencyStorage { + Decision: Store annotation frequencies as absolute values (center frequency plus the relative offset). + + Scope: Governs frequency bounds on all annotations, including those produced by the parallel separator. + + Design issues addressed: Relative-only frequencies are ambiguous once a recording is shifted to baseband or compared across recordings. + + Consequences: The parallel separator works in baseband and converts its results to absolute frequencies before storing them. + } + } } diff --git a/src/ria_toolkit_oss/app/app.sigil b/src/ria_toolkit_oss/app/app.sigil index d1917e3..7e9407d 100644 --- a/src/ria_toolkit_oss/app/app.sigil +++ b/src/ria_toolkit_oss/app/app.sigil @@ -1,24 +1,45 @@ component AppRunner { goal { Pull and run containerized RIA applications, auto-configuring hardware access from image labels. + Own the ria-app CLI over a container engine. } interface { - CLI ria-app with subcommands: pull, run, list, stop, logs, and configure; a global --sudo flag. - run auto-pulls the image if absent and derives container flags from the image's ria.* labels. + AppCommands { + CLI ria-app with subcommands: pull, run, list, stop, logs, and configure; a global --sudo flag. + } + + RunBehavior { + run auto-pulls the image if absent and derives container flags from the image's ria.* labels. + } } } expand AppRunner { logic { The engine is the first of docker or podman on PATH, optionally sudo-prefixed. + Image references resolve against a configured registry and namespace; fully-qualified references pass through. + Hardware flags are derived from ria.profile and ria.hardware labels: GPU access when an NVIDIA profile and runtime are present, USB device passthrough for USB SDRs, and host networking for USRP, ThinkRF, and Pluto; flags can be overridden or dry-run. } constraints { There is no Python SDK; it is a subprocess wrapper and exits with an error if no container engine is found. + Config is AppConfig (registry, namespace, sudo) at ~/.ria/toolkit.json with environment-variable fallbacks. } + + decisions { + SubprocessWrapper { + Decision: ria-app shells out to a container engine (the first of docker or podman on PATH) rather than exposing a Python SDK or using an engine client library. + + Scope: Governs how the runner controls containers. + + Consequences: A container engine must be present on PATH and the tool exits with an error when none is found; there is no in-process programmatic API for callers. + + Trade-offs: Reuses the installed engine and its existing auth and configuration directly at the cost of no library-level integration point. + } + } } diff --git a/src/ria_toolkit_oss/data/data.sigil b/src/ria_toolkit_oss/data/data.sigil index 7018659..e24275c 100644 --- a/src/ria_toolkit_oss/data/data.sigil +++ b/src/ria_toolkit_oss/data/data.sigil @@ -1,34 +1,83 @@ component RadioData { goal { Provide the core radio-ML data types: a Recording (an immutable IQ tape with metadata and annotations) and the radio dataset and dataset-builder framework. + Own the in-memory contract the rest of the toolkit and RIA Hub build on for signals and datasets. } interface { - Recording(data, metadata=None, dtype=None, timestamp=None, annotations=None) - complex IQ data shaped channels-by-samples (1-D is reshaped to one channel); metadata must be a JSON-serializable dict; rec_id and timestamp are auto-populated. - Recording properties: data, metadata, annotations, shape, n_chan, rec_id, dtype, timestamp, sample_rate (settable). - Recording methods: astype, add_to_metadata/update_metadata/remove_from_metadata (rec_id and timestamp protected), trim, normalize, view/simple_view, and to_sigmf/to_npy/to_wav/to_blue (delegating to io). - Annotation(sample_start, sample_count, freq_lower_edge=None, freq_upper_edge=None, label="", comment="", detail=None) with is_valid, overlap, area, and to_sigmf_format. - Dataset framework (data.datasets): abstract RadioDataset (HDF5-backed and iterable; item assignment raises, though it exposes explicit inplace mutators) with IQDataset and SpectDataset subclasses, DatasetBuilder, and leakage-safe split/random_split. + Recording { + Recording(data, metadata=None, dtype=None, timestamp=None, annotations=None) - complex IQ data shaped channels-by-samples (1-D is reshaped to one channel); metadata must be a JSON-serializable dict; rec_id and timestamp are auto-populated. + + Recording properties: data, metadata, annotations, shape, n_chan, rec_id, dtype, timestamp, sample_rate (settable). + + Recording methods: astype, add_to_metadata/update_metadata/remove_from_metadata (rec_id and timestamp protected), trim, normalize, view/simple_view, and to_sigmf/to_npy/to_wav/to_blue (delegating to io). + } + + Annotation { + Annotation(sample_start, sample_count, freq_lower_edge=None, freq_upper_edge=None, label="", comment="", detail=None) with is_valid, overlap, area, and to_sigmf_format. + } + + DatasetFramework { + Dataset framework (data.datasets): abstract RadioDataset (HDF5-backed and iterable; item assignment raises, though it exposes explicit inplace mutators) with IQDataset and SpectDataset subclasses, DatasetBuilder, and leakage-safe split/random_split. + } } } expand RadioData { state { A Recording's sample data is immutable: item assignment raises, and property accessors return copies or read-only views; metadata is mutable only through the metadata methods. + Metadata keys must be lowercase letters and underscores only (no digits); rec_id and timestamp are protected and cannot be modified or removed. } logic { A Recording requires complex data of one or two dimensions (channels by samples); rec_id is the SHA-256 of the data bytes plus timestamp. + trim slices samples and re-aligns annotations; normalize scales peak magnitude to 1 and rejects all-zero data. + RadioDataset reads data and metadata lazily from an HDF5 source; non-inplace mutations write numbered sibling files; concrete backends live outside the OSS package. + split and random_split partition by rec_id so no recording leaks across subsets; lengths may be counts or fractions. } constraints { Recording data must be complex or construction raises; metadata must be a JSON-serializable dict. + RadioDataset sources must be HDF5 (validated at construction); the data and metadata datasets are required for use but not checked at init; the class is abstract and instantiated only via a concrete backend. + The datatypes package is stale (only bytecode remains); the live location is data. } + + decisions { + RecordingImmutability { + Decision: Make Recording sample data immutable and derive rec_id as the SHA-256 of the data bytes plus timestamp. + + Scope: Governs Recording identity and the sample-data contract; metadata remains mutable through the dedicated metadata methods. + + Design issues addressed: A content-addressed identity is stable only if the underlying samples cannot change after construction, so item assignment raises and accessors return copies or read-only views. + + Consequences: Datasets and splits can key on rec_id as a durable identity, but any modification must produce a new Recording rather than mutate in place. + } + + LeakageSafeSplit { + Decision: Partition split and random_split by rec_id so no recording appears in more than one subset. + + Scope: Governs dataset splitting for train/validation/test construction. + + Design issues addressed: Sharing a recording across subsets leaks information between ML training and evaluation sets. + + Assumptions: rec_id uniquely identifies a recording's samples. + } + + HdfBackedAbstractDataset { + Decision: Keep RadioDataset abstract and HDF5-backed, with concrete storage backends living outside the OSS package. + + Scope: Governs the dataset framework's storage contract; IQDataset and SpectDataset define the data shape, backends define storage. + + Design issues addressed: Large radio datasets need lazy on-disk reads, and the OSS package should define the dataset contract without shipping a specific concrete backend. + + Consequences: HDF5 sources are validated at construction, and the class cannot be instantiated directly without a concrete backend. + } + } } diff --git a/src/ria_toolkit_oss/io/io.sigil b/src/ria_toolkit_oss/io/io.sigil index c29cdb0..75552db 100644 --- a/src/ria_toolkit_oss/io/io.sigil +++ b/src/ria_toolkit_oss/io/io.sigil @@ -1,28 +1,55 @@ component RecordingIO { goal { Load and save Recording objects to and from file across the RF formats the toolkit supports. + Own format detection and the per-format readers and writers. } interface { - load_recording(file) - dispatch on file extension to the matching reader (sigmf, npy, wav, blue); an unknown extension raises. - Per-format readers and writers: to_sigmf/from_sigmf, to_npy/from_npy (plus from_npy_legacy), to_wav/from_wav, to_blue/from_blue. - Most writers return the output path (to_sigmf returns None) and honor an overwrite flag; auto-generated filenames land under recordings/. + FormatDispatch { + load_recording(file) - dispatch on file extension to the matching reader (sigmf, npy, wav, blue); an unknown extension raises. + } + + FormatCodecs { + Per-format readers and writers: to_sigmf/from_sigmf, to_npy/from_npy (plus from_npy_legacy), to_wav/from_wav, to_blue/from_blue. + } + + WriterBehavior { + Most writers return the output path (to_sigmf returns None) and honor an overwrite flag; auto-generated filenames land under recordings/. + } } } expand RecordingIO { logic { The npy v2 format stores a version marker then data, JSON metadata, and JSON annotations, avoiding pickle; legacy npy paths fall back to pickle with a warning and normalize namespaced metadata keys. + SigMF writes paired .sigmf-data and .sigmf-meta, mapping RIA metadata to core keys and namespacing the rest under ria:. + WAV stores I and Q as stereo channels with metadata in an info chunk; Blue uses the MIDAS 512-byte header format. } constraints { Format detection is by file extension only. + The SigMF, WAV, and Blue writers reject multichannel input and the readers produce single-channel output; only npy round-trips multichannel data. + Integer-encoded outputs (16-bit WAV, integer Blue) require samples normalized to the range -1 to 1. + io.common (exists, validate, move, copy) is declared but unimplemented. + save_recording is exported in the package __all__ but not defined (latent bug). } + + decisions { + NpyPickleAvoidance { + Decision: Store the npy v2 format as a version marker followed by data, JSON metadata, and JSON annotations rather than a pickle. + + Scope: Governs the native npy reader and writer; legacy npy files still load through a pickle fallback. + + Design issues addressed: Pickle payloads are unsafe to load from untrusted sources and are not portable across Python versions. + + Consequences: Legacy npy paths fall back to pickle with a warning and normalize namespaced metadata keys for backward compatibility. + } + } } diff --git a/src/ria_toolkit_oss/orchestration/orchestration.sigil b/src/ria_toolkit_oss/orchestration/orchestration.sigil index d9491a8..67217f8 100644 --- a/src/ria_toolkit_oss/orchestration/orchestration.sigil +++ b/src/ria_toolkit_oss/orchestration/orchestration.sigil @@ -1,26 +1,49 @@ component Orchestration { goal { Define and execute automated RF capture campaigns: sweep transmitters and channels, record, QA, and label the results. + Own the campaign configuration schema, the executor, and the QA checks. } interface { - CampaignConfig(name, recorder, transmitters, qa, output, mode, loops) with loaders from_dict, from_yaml, and from_device_profile; it requires a recorder and at least one transmitter. - Supporting dataclasses: RecorderConfig, TransmitterConfig (type and control_method external_script/sdr/sdr_remote/sdr_agent), CaptureStep, QAConfig, and OutputConfig. - CampaignExecutor(config, progress_cb=None).run() returns a CampaignResult of StepResults; check_recording(recording, config) returns a QAResult; label_recording writes ria: SigMF metadata. + CampaignSchema { + CampaignConfig(name, recorder, transmitters, qa, output, mode, loops) with loaders from_dict, from_yaml, and from_device_profile; it requires a recorder and at least one transmitter. + + Supporting dataclasses: RecorderConfig, TransmitterConfig (type and control_method external_script/sdr/sdr_remote/sdr_agent), CaptureStep, QAConfig, and OutputConfig. + } + + Execution { + CampaignExecutor(config, progress_cb=None).run() returns a CampaignResult of StepResults; check_recording(recording, config) returns a QAResult; label_recording writes ria: SigMF metadata. + } } } expand Orchestration { logic { The executor opens the SDR and remote TX controllers once, then for each loop, transmitter, and step it starts the transmitter, records IQ, stops the transmitter, then labels, QA-checks, and saves it as SigMF. + QA estimates SNR from the PSD and checks duration and SNR; in flag-for-review mode a recording always passes but may be flagged. + Cancellation is cooperative at step boundaries via the progress callback. } constraints { Frequencies, gains, durations, and bandwidths are parsed from human units (for example 2.45GHz, auto, 1.5m). + Device names are aliased onto SDR drivers, including mock and sim for hardware-free runs. + The controller's Conductor and the RT-OSS server are parallel HTTP wrappers that both reuse this package's CampaignConfig and CampaignExecutor in-process. } + + decisions { + SingleExecutionCore { + Decision: Campaign configuration and execution live in this package as a wrapper-agnostic in-process core, reused directly by both the controller's Conductor and the RT-OSS server rather than reimplemented behind each HTTP front end. + + Scope: Governs where campaign orchestration logic lives relative to its HTTP wrappers. + + Consequences: The HTTP wrappers stay thin and share identical campaign semantics; a change to campaign behavior here propagates to every wrapper. + + Trade-offs: Both wrappers are coupled to this package's in-process API and dataclasses rather than to a stable network contract. + } + } } diff --git a/src/ria_toolkit_oss/remote_control/remote-control.sigil b/src/ria_toolkit_oss/remote_control/remote-control.sigil index 5e93fbb..432aaba 100644 --- a/src/ria_toolkit_oss/remote_control/remote-control.sigil +++ b/src/ria_toolkit_oss/remote_control/remote-control.sigil @@ -1,25 +1,47 @@ component RemoteControl { goal { Drive a software-defined radio transmitter on a remote machine over SSH and ZMQ. + Own the client controller and the remote server that bridge transmit commands to a remote SDR. } interface { - RemoteTransmitterController(host, ssh_user, ssh_key_path, zmq_port=5556): set_radio, init_tx, transmit_async, wait_transmit, stop, close. - RemoteTransmitter (the server on the transmit machine): set_radio, init_tx, transmit, stop, and a run_function JSON-RPC dispatcher; launched as a module over a ZMQ REP socket. + TransmitterController { + RemoteTransmitterController(host, ssh_user, ssh_key_path, zmq_port=5556): set_radio, init_tx, transmit_async, wait_transmit, stop, close. + } + + TransmitterServer { + RemoteTransmitter (the server on the transmit machine): set_radio, init_tx, transmit, stop, and a run_function JSON-RPC dispatcher; launched as a module over a ZMQ REP socket. + } } } expand RemoteControl { logic { The controller SSHes into the transmit host (paramiko, key-file auth), launches the remote_transmitter server there, waits for it to bind, then drives it with JSON-RPC over a ZMQ REQ/REP socket. + Supported remote devices are the transmit-capable drivers: Pluto, USRP, HackRF, and BladeRF. + set_radio must precede init_tx and transmit (both guard on a radio being set); the init_tx-before-transmit order is a usage convention, not an enforced guard; transmit runs in a daemon thread joined by wait_transmit. } constraints { SSH host-key policy is AutoAddPolicy, which accepts unknown hosts (security note). + RIA Hub does not use this subpackage; remote and agent transmit is handled through the agent path instead. + Known gap: transmit calls an SDR method tx_cw that no driver defines, so remote CW transmit currently no-ops for real drivers (the AttributeError is swallowed). } + + decisions { + HostKeyAutoAdd { + Decision: SSH connections use paramiko AutoAddPolicy, accepting an unknown host key on first connect instead of requiring pre-provisioned known hosts. + + Scope: Governs how the controller establishes the SSH channel to the transmit host. + + Trade-offs: Removes manual known-hosts setup for ad-hoc transmit hosts at the cost of exposure to man-in-the-middle on the initial connection. + + Consequences: Host authenticity is not verified, so the SSH channel's integrity depends on the surrounding network being trusted. + } + } } diff --git a/src/ria_toolkit_oss/sdr/sdr.sigil b/src/ria_toolkit_oss/sdr/sdr.sigil index 7ca0893..ec65374 100644 --- a/src/ria_toolkit_oss/sdr/sdr.sigil +++ b/src/ria_toolkit_oss/sdr/sdr.sigil @@ -1,29 +1,47 @@ component SdrInterface { goal { Provide a unified receive and transmit API across a variety of software-defined radios. + Own the SDR base contract, the per-device drivers, and device discovery and selection. } interface { - get_sdr_device(device_type, ident=None, tx=False) returns an SDR; mock and sim return a MockSDR, real devices delegate to the CLI package's factory. - detect_available() returns the drivers whose optional dependency imports cleanly; it does not probe hardware. - SDR base class: record(num_samples, rx_time) and rx(num_samples) to receive, tx_recording(recording) to transmit, stream_to_zmq and pickle_buffer_to_zmq to stream, getters and setters for sample rate, center frequency, and gain, and capability probes for bias-tee and dynamic-update support. - Drivers: Pluto, USRP, HackRF, and Blade (receive and transmit); RTLSDR and ThinkRF (receive only); MockSDR (AWGN generator, no hardware, receive and transmit). Real drivers are constructed with a device identifier; MockSDR takes a buffer size and seed. - Errors: SDRError with SDRParameterError and SdrDisconnectedError (SDROverflowError is defined but not in the public __all__). + DeviceDiscovery { + get_sdr_device(device_type, ident=None, tx=False) returns an SDR; mock and sim return a MockSDR, real devices delegate to the CLI package's factory. + + detect_available() returns the drivers whose optional dependency imports cleanly; it does not probe hardware. + } + + SdrContract { + SDR base class: record(num_samples, rx_time) and rx(num_samples) to receive, tx_recording(recording) to transmit, stream_to_zmq and pickle_buffer_to_zmq to stream, getters and setters for sample rate, center frequency, and gain, and capability probes for bias-tee and dynamic-update support. + } + + Drivers { + Drivers: Pluto, USRP, HackRF, and Blade (receive and transmit); RTLSDR and ThinkRF (receive only); MockSDR (AWGN generator, no hardware, receive and transmit). Real drivers are constructed with a device identifier; MockSDR takes a buffer size and seed. + } + + SdrErrors { + Errors: SDRError with SDRParameterError and SdrDisconnectedError (SDROverflowError is defined but not in the public __all__). + } } } expand SdrInterface { logic { init_rx or init_tx must be called before record or tx_recording (rx auto-initializes on first use); they push sample rate, center frequency, and gain to the hardware via per-driver setters. + Receive accumulates validated buffers into a Recording; transmit loops or truncates the source IQ to the requested length. + A driver is available only if its optional hardware dependency imports; USB or network drops are translated to SdrDisconnectedError. } constraints { gain_mode absolute applies the requested gain (clamped or snapped to the device's valid range); relative requires a negative (attenuation) value; ThinkRF uses named gain profiles instead. + RTLSDR and ThinkRF are receive-only; their transmit methods raise or are stubs. + Identifiers are device-specific free-form values (Pluto URI or IP, USRP serial or args, HackRF/BladeRF serial). + RIA Hub consumes this layer through the SDR base contract and get_sdr_device but currently wires only Pluto and USRP live; the controller imports driver modules directly rather than only via the factory. } } diff --git a/src/ria_toolkit_oss/server/server.sigil b/src/ria_toolkit_oss/server/server.sigil index 04a1f89..4329b08 100644 --- a/src/ria_toolkit_oss/server/server.sigil +++ b/src/ria_toolkit_oss/server/server.sigil @@ -1,28 +1,63 @@ component RtOssServer { goal { Expose the toolkit's campaign orchestration and live RF inference over HTTP as a standalone service (RT-OSS). + Own the FastAPI app, its auth, and the deploy and inference endpoints. } interface { - create_app(api_key="") returns a FastAPI app mounting /conductor and /inference (both API-key gated) plus an unauthenticated GET /health; an empty api_key disables auth (dev only). - serve() launches uvicorn from RT_OSS_API_KEY, RT_OSS_HOST (default 0.0.0.0), and RT_OSS_PORT (default 8080), and is the ria-server entry point; the ria serve CLI command is separate and calls create_app() directly. - Conductor endpoints: POST /conductor/deploy ({config}) returns {campaign_id} and runs the campaign in a background thread; GET /conductor/status/{campaign_id}; POST /conductor/cancel/{campaign_id}. - Inference endpoints: POST /inference/load (ONNX model plus label_map), POST /inference/start, POST /inference/stop, POST /inference/configure, GET /inference/status. + AppFactory { + create_app(api_key="") returns a FastAPI app mounting /conductor and /inference (both API-key gated) plus an unauthenticated GET /health; an empty api_key disables auth (dev only). + + serve() launches uvicorn from RT_OSS_API_KEY, RT_OSS_HOST (default 0.0.0.0), and RT_OSS_PORT (default 8080), and is the ria-server entry point; the ria serve CLI command is separate and calls create_app() directly. + } + + ConductorEndpoints { + Conductor endpoints: POST /conductor/deploy ({config}) returns {campaign_id} and runs the campaign in a background thread; GET /conductor/status/{campaign_id}; POST /conductor/cancel/{campaign_id}. + } + + InferenceEndpoints { + Inference endpoints: POST /inference/load (ONNX model plus label_map), POST /inference/start, POST /inference/stop, POST /inference/configure, GET /inference/status. + } } } expand RtOssServer { logic { A deploy parses the body via CampaignConfig.from_dict and rejects any transmitter carrying a script field (external scripts blocked). + The inference loop captures 4096 IQ samples, estimates SNR, preprocesses to the model input shape, softmaxes, and maps the index to a label; idle-class labels are reported as idle. + State is in-memory only (a campaign dict plus a single thread-locked inference state); nothing is persisted. } constraints { Auth is X-API-Key compared in constant time; a missing configured key allows all requests (dev only). + Server dependencies (fastapi, uvicorn, onnxruntime) are an optional extra; a missing onnxruntime or SDR yields 503. + RIA Hub's controller does not call this server; it embeds the toolkit library in-process. This server is the alternative standalone deployment. + + A deploy rejects any transmitter carrying a script field; external scripts are blocked. + model_path is path-traversal validated and must resolve to a .onnx file. } + + decisions { + StandaloneDeployment { + Decision: RT-OSS is packaged as a standalone HTTP service; RIA Hub's controller does not call it but embeds the toolkit library in-process instead. + + Scope: Governs how this server relates to the primary RIA Hub deployment. + + Consequences: The server carries its own auth, in-memory state, and process lifecycle rather than sharing the controller's, and exists as the alternative deployment for environments that want the toolkit behind HTTP. + } + + ScriptTransmitterRejection { + Decision: Deploy rejects any transmitter config carrying a script field, blocking external scripts. + + Scope: Governs the campaign config accepted by POST /conductor/deploy. + + Design issues addressed: Honoring a script field on a network-exposed, API-key-gated deploy endpoint would allow a caller to run arbitrary external commands on the server host. + } + } } diff --git a/src/ria_toolkit_oss/signal/signal.sigil b/src/ria_toolkit_oss/signal/signal.sigil index 7bf160e..a68e44b 100644 --- a/src/ria_toolkit_oss/signal/signal.sigil +++ b/src/ria_toolkit_oss/signal/signal.sigil @@ -1,26 +1,63 @@ component SignalProcessing { goal { Provide the toolkit's signal generation and processing suite: functional waveform generators and a modular block-based modulation-chain framework. + Own the DSP building blocks that produce Recording objects for datasets and transmission. } interface { - Functional generators returning a Recording: sine, square, sawtooth, noise, chirp, lfm_chirp_complex, complex_sine. - Block framework: Block, SourceBlock, ProcessBlock, and RecordableBlock forming a DataType-typed chain, with sources (BinarySource, AWGNSource, SineSource, RecordingSource, ...), Mapper (PSK/QAM/PAM constellations; standalone APSK and cross-QAM mapper classes exist but are not reachable through Mapper) and SymbolDemapper, symbol modulators (GMSK, OOK, OQPSK), continuous-phase modulators (FSK, CPFSK), pulse-shaping filters (RaisedCosine, RootRaisedCosine, Gaussian, Sinc, Rect), Upsampling, channels (AWGNChannel, FlatRayleigh), and basic ops (Add, MultiplyConstant, PhaseShift, FrequencyShift, frequency up/down conversion). - High-level generators: PSKGenerator, QAMGenerator, and PAMGenerator, each with record(batch_size, num_bits). - This layer is a standalone library surface; the RIA Hub controller does not import it directly. + FunctionalGenerators { + Functional generators returning a Recording: sine, square, sawtooth, noise, chirp, lfm_chirp_complex, complex_sine. + } + + BlockFramework { + Block framework: Block, SourceBlock, ProcessBlock, and RecordableBlock forming a DataType-typed chain, with sources (BinarySource, AWGNSource, SineSource, RecordingSource, ...), Mapper (PSK/QAM/PAM constellations; standalone APSK and cross-QAM mapper classes exist but are not reachable through Mapper) and SymbolDemapper, symbol modulators (GMSK, OOK, OQPSK), continuous-phase modulators (FSK, CPFSK), pulse-shaping filters (RaisedCosine, RootRaisedCosine, Gaussian, Sinc, Rect), Upsampling, channels (AWGNChannel, FlatRayleigh), and basic ops (Add, MultiplyConstant, PhaseShift, FrequencyShift, frequency up/down conversion). + } + + HighLevelGenerators { + High-level generators: PSKGenerator, QAMGenerator, and PAMGenerator, each with record(batch_size, num_bits). + } + + StandaloneSurface { + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } } } expand SignalProcessing { logic { A SignalGenerator validates that each block's output type matches the next block's input type; sample production is driven by the blocks themselves (Block and RecordableBlock via get_samples, and the PSK/QAM/PAM generators by chaining block __call__). + Blocks auto-serialize their JSON-safe attributes into the produced Recording's metadata. } constraints { Supported modulations reachable via Mapper and the generators: PSK, QAM, PAM (constellation); GMSK, OOK, OQPSK (symbol); FSK and CPFSK (continuous). QAM and PAM require an even bits-per-symbol (QAM additionally more than 2). APSK and cross-QAM mapper classes exist but are unwired. + The basic generators (sine, square, sawtooth, noise, complex_sine) raise on a sample rate below 1 (chirp instead requires at least two samples); FSK and CPFSK require frequency spacing times symbol duration of at least 0.5. + Data flows as typed stages (bits, symbols, upsampled symbols, baseband, passband) via the DataType enum. } + + decisions { + TypedBlockChain { + Decision: Model the modulation chain as DataType-typed blocks and validate that each block's output type matches the next block's input type. + + Scope: Governs the block framework and the SignalGenerator that assembles blocks; the functional generators bypass the chain. + + Design issues addressed: Connecting incompatible DSP stages (for example feeding symbols where baseband is expected) would otherwise fail silently or produce meaningless output. + + Consequences: Data flows as explicit typed stages (bits, symbols, upsampled symbols, baseband, passband) via the DataType enum. + } + + BlockMetadataProvenance { + Decision: Have blocks auto-serialize their JSON-safe attributes into the produced Recording's metadata. + + Scope: Governs metadata written by block-based generation. + + Design issues addressed: A generated Recording should carry the parameters that produced it so datasets are reproducible and self-describing. + + Assumptions: The block attributes relevant to reproduction are JSON-serializable. + } + } } diff --git a/src/ria_toolkit_oss/transforms/transforms.sigil b/src/ria_toolkit_oss/transforms/transforms.sigil index 3714156..5b3d16b 100644 --- a/src/ria_toolkit_oss/transforms/transforms.sigil +++ b/src/ria_toolkit_oss/transforms/transforms.sigil @@ -1,21 +1,47 @@ component Transforms { goal { Provide NumPy radio-data transforms used by the ML backends: data augmentations and channel/hardware impairment models over IQ. + Own the functions that manipulate a signal while preserving its container type. } interface { - Augmentations (iq_augmentations): generate_awgn, time_reversal, spectral_inversion, channel_swap, amplitude_reversal, drop_samples, quantize_tape, quantize_parts, magnitude_rescale, cut_out, patch_shuffle. - Impairments (iq_impairments): add_awgn_to_signal, time_shift, frequency_shift, phase_shift, iq_imbalance, resample. - Every function accepts an array or a Recording and returns the same type, preserving metadata for a Recording. - This layer is a standalone library surface; the RIA Hub controller does not import it directly. + Augmentations { + Augmentations (iq_augmentations): generate_awgn, time_reversal, spectral_inversion, channel_swap, amplitude_reversal, drop_samples, quantize_tape, quantize_parts, magnitude_rescale, cut_out, patch_shuffle. + } + + Impairments { + Impairments (iq_impairments): add_awgn_to_signal, time_shift, frequency_shift, phase_shift, iq_imbalance, resample. + } + + TypePreservingContract { + Every function accepts an array or a Recording and returns the same type, preserving metadata for a Recording. + } + + StandaloneSurface { + This layer is a standalone library surface; the RIA Hub controller does not import it directly. + } } } expand Transforms { constraints { Input must be a 2-D complex channels-by-samples array or a ValueError is raised; most transforms implement only the single-channel path and raise NotImplementedError for more channels. + frequency_shift is relative to the sample rate and must be within -0.5 to 0.5; phase_shift must be within -pi to pi. + AWGN is generated to match a target SNR in dB. } + + decisions { + TypePreservingTransforms { + Decision: Accept either a raw array or a Recording and return the same type, preserving metadata when the input is a Recording. + + Scope: Governs the call contract of every augmentation and impairment function. + + Design issues addressed: ML pipelines mix bare IQ arrays and Recording objects, and a transform must not strip a Recording's metadata as it passes through. + + Consequences: Callers can compose transforms without branching on input type or re-attaching metadata. + } + } } diff --git a/src/ria_toolkit_oss_cli/cli.sigil b/src/ria_toolkit_oss_cli/cli.sigil index e7de867..582d386 100644 --- a/src/ria_toolkit_oss_cli/cli.sigil +++ b/src/ria_toolkit_oss_cli/cli.sigil @@ -1,11 +1,17 @@ component ToolkitCli { goal { Provide the ria and ria-tools command-line interface over the toolkit's library features. + Own the Click command group and its subcommands. } interface { - A Click group that auto-registers the commands defined in ria_toolkit_oss_cli.ria_toolkit_oss.commands and warns when git-lfs is missing. - Commands map to library features: capture and transmit (sdr), campaign (orchestration), generate and synth (signal), transform/combine/split/convert (transforms, data, io), annotate (annotations), view (view and viz), discover (sdr), serve (the RT-OSS server), and upload/setup_repo/init (RIA Hub repo integration). + CommandGroup { + A Click group that auto-registers the commands defined in ria_toolkit_oss_cli.ria_toolkit_oss.commands and warns when git-lfs is missing. + } + + CommandMap { + Commands map to library features: capture and transmit (sdr), campaign (orchestration), generate and synth (signal), transform/combine/split/convert (transforms, data, io), annotate (annotations), view (view and viz), discover (sdr), serve (the RT-OSS server), and upload/setup_repo/init (RIA Hub repo integration). + } } } -- 2.34.1