Training run - 2026-05-28 11:52:05

Copied from qoherent/ash_test_demo_repo/good/ for the WavesFM ICC demo.
SigMF data + meta pairs, ~27 MB total. Use these as input to the Curator.

.riahub/train_configs/model/model.yaml

@@ -0,0 +1,5 @@
+source: wavesfm
+name: WavesFM Linear Probe
+task: rml
+epochs: 5
+batch_size: 2048

.riahub/workflows/train.yaml

@@ -15,11 +15,11 @@ permissions:
 
 jobs:
   WavesFM-Training:
-    runs-on: "ubuntu-latest"
+    runs-on: "ubuntu-24.04"
     env:
       WAVESFM_TASK: "rml"
-      WAVESFM_EPOCHS: "3"
-      WAVESFM_BATCH_SIZE: "16"
+      WAVESFM_EPOCHS: "5"
+      WAVESFM_BATCH_SIZE: "2048"
       WAVESFM_OUTPUT_DIR: "/opt/wavesfm/output"
       # Single source of truth for the cloned WavesFM repo location.
       # Referenced as ${{ env.WAVESFM_REPO_DIR }} in steps. To relocate
@@ -27,6 +27,8 @@ jobs:
       # downstream step uses the env var, no hard-coded paths.
       WAVESFM_REPO_DIR: "/opt/wavesfm/repo"
       WAVESFM_ADAPTED_DATA: "/opt/wavesfm/adapted_data.h5"
+      # Override model-download base URL to internal LAN IP
+      # (external riahub.ai LFS endpoint is unreachable from runners).
       RIAHUB_BASE_URL: "http://192.168.0.170:3000"
     steps:
       - name: Display basic runner info
@@ -42,7 +44,6 @@ jobs:
             echo "No NVIDIA GPU available."
           fi
 
-
       - name: "Download Model (qoherent/wavesfm-base/wavesfm-v1p0.pth)"
         shell: bash
         timeout-minutes: 4
@@ -179,8 +180,7 @@ jobs:
             fi
             exit 1
           fi
-
-      - name: "Checkout Dataset (qoherent/icc-demo/icc_canary_2026_05_28-v1.0.0.h5)"
+      - name: Checkout Training Dataset
         shell: bash
         timeout-minutes: 10
         env:
@@ -210,16 +210,8 @@ jobs:
 
           AUTH_HEADER=""
           if [[ -n "${RIAHUB_USER:-}" && -n "${RIAHUB_TOKEN:-}" ]]; then
-            AUTH_HEADER=$(printf 'Authorization: basic %s' \
-              "$(printf '%s:%s' "$RIAHUB_USER" "$RIAHUB_TOKEN" | base64 | tr -d '\n')")
+            AUTH_HEADER=$(printf 'Authorization: basic %s' "$(printf '%s:%s' "$RIAHUB_USER" "$RIAHUB_TOKEN" | base64 | tr -d '\n')")
           fi
-          # ``sudo env GIT_TERMINAL_PROMPT=0`` propagates the env var across
-          # sudo's default ``env_reset`` boundary; a bare ``sudo git`` would
-          # see an empty env on most distros' default sudoers, so the
-          # step-level ``env:`` block's GIT_TERMINAL_PROMPT=0 would NOT
-          # actually reach git child processes. Without it, git falls back
-          # to opening ``/dev/tty`` (the PTY allocated by act_runner) and
-          # prompting for credentials on a 401, hanging until timeout.
           git_auth() {
             if [[ -n "$AUTH_HEADER" ]]; then
               sudo env GIT_TERMINAL_PROMPT=0 git -c "http.extraheader=$AUTH_HEADER" "$@"
@@ -227,9 +219,8 @@ jobs:
               sudo env GIT_TERMINAL_PROMPT=0 git "$@"
             fi
           }
-
           REPO_PATH='/qoherent/icc-demo.git'
-          DEST_ROOT='/opt/qmb/riahub/dataset/qoherent/icc-demo/9f87fa9fe2badd314ad81379064e236ea494e89d'
+          DEST_ROOT='/opt/qmb/riahub/dataset/qoherent/icc-demo/main'
           sudo mkdir -p "$(dirname "$DEST_ROOT")"
           if ! command -v git-lfs >/dev/null 2>&1; then
             sudo apt-get update -y
@@ -245,31 +236,16 @@ jobs:
             sudo mkdir -p "$DEST_ROOT"
             sudo git -C "$DEST_ROOT" init || continue
             sudo git -C "$DEST_ROOT" remote add origin "$REPO_URL" || continue
-            # See ``_render_model_checkout`` for the rationale on skipping
-            # ``git lfs install --local`` — short version: the smudge
-            # filter it would register tries its own credential lookup
-            # during ``git checkout FETCH_HEAD`` and hangs forever on
-            # /dev/tty when the repo is internal/private. We rely on
-            # the explicit ``git lfs fetch`` (with auth) +
-            # ``git lfs checkout`` (local) pair below instead.
             sudo git -C "$DEST_ROOT" sparse-checkout init --no-cone || continue
             sudo git -C "$DEST_ROOT" sparse-checkout set --no-cone -- \
-              'icc_canary_2026_05_28-v1.0.0.h5' || continue
-            if ! git_auth -C "$DEST_ROOT" fetch --depth=1 origin '9f87fa9fe2badd314ad81379064e236ea494e89d'; then
+              'datasets/icc28-train_v1.0.0.h5' || continue
+            if ! git_auth -C "$DEST_ROOT" fetch --depth=1 origin 'ef0a0e430f8e5019dec52794fccab958b3a3c2b7'; then
               continue
             fi
-            # See ``_render_model_checkout`` for the rationale on
-            # ``GIT_LFS_SKIP_SMUDGE=1`` — short version: the runner has
-            # the LFS smudge filter installed system-wide
-            # (``/etc/gitconfig``), so checkout fires it and the filter's
-            # credential helper hangs on /dev/tty for internal repos.
-            # Skipping smudge here lets the explicit ``git lfs fetch``
-            # below handle materialization with proper auth.
-            if ! sudo env GIT_TERMINAL_PROMPT=0 GIT_LFS_SKIP_SMUDGE=1 \
-                git -C "$DEST_ROOT" -c advice.detachedHead=false checkout FETCH_HEAD; then
+            if ! sudo env GIT_TERMINAL_PROMPT=0 GIT_LFS_SKIP_SMUDGE=1 git -C "$DEST_ROOT" -c advice.detachedHead=false checkout FETCH_HEAD; then
               continue
             fi
-            if ! git_auth -C "$DEST_ROOT" lfs fetch origin --include='icc_canary_2026_05_28-v1.0.0.h5' --exclude=""; then
+            if ! git_auth -C "$DEST_ROOT" lfs fetch origin --include='datasets/icc28-train_v1.0.0.h5' --exclude=""; then
               echo "LFS fetch failed for candidate $base, trying next" >&2
               continue
             fi
@@ -278,7 +254,7 @@ jobs:
               continue
             fi
             POINTER_FOUND=0
-            _LFS_REL_PATH='icc_canary_2026_05_28-v1.0.0.h5'
+            _LFS_REL_PATH='datasets/icc28-train_v1.0.0.h5'
             if [[ "$(sudo head -c 9 "$DEST_ROOT/$_LFS_REL_PATH" 2>/dev/null || true)" == "version h" ]]; then
               echo "LFS materialization left a pointer at $DEST_ROOT/$_LFS_REL_PATH for candidate $base, trying next" >&2
               POINTER_FOUND=1
@@ -292,7 +268,7 @@ jobs:
           if [[ "$MATERIALIZED" -ne 1 ]]; then
             echo "Failed to materialize dataset using base URL candidates derived from: $BASE_URL_SOURCE" >&2
             if [[ -z "$AUTH_HEADER" ]]; then
-              echo "  (set QMBDEMO_USER+QMBDEMO_TOKEN repo secrets for internal/private repos)" >&2
+              echo "  (no credentials configured: set QMBDEMO_USER and QMBDEMO_TOKEN repo secrets for internal/private repos)" >&2
             fi
             exit 1
           fi
@@ -321,7 +297,7 @@ jobs:
           # `--device cpu` from the Train step actually takes effect.
           # No-op if the line already uses args.device (idempotent).
           if [[ -f main_finetune.py ]]; then
-            sed -i 's|torch\.amp\.GradScaler(device="cuda")|torch.amp.GradScaler(device=args.device, enabled=(args.device != "cpu"))|' main_finetune.py
+            sed -i 's|torch\.amp\.GradScaler(device="cuda")|torch.cuda.amp.GradScaler(enabled=(args.device != "cpu"))|' main_finetune.py
             echo "Patched main_finetune.py GradScaler for CPU/GPU device parity."
           fi
 
@@ -358,11 +334,9 @@ jobs:
           # only when the repo is genuinely installable (has setup.py /
           # setup.cfg, or pyproject.toml with [build-system]).
           cd "$WAVESFM_REPO_DIR"
-          # FAST-PATH: install CPU torch from pytorch.org/whl/cpu FIRST (~200MB).
-          # This makes torch==X already-satisfied so requirements.txt does not
-          # pull the 755MB manylinux wheel with bundled CUDA from PyPI default.
+          # Pre-install CPU torch + numpy<2 to make requirements.txt see them already-satisfied (saves ~600MB).
+          # torch 2.2.2 has the NumPy 1.x ABI and crashes if numpy 2.x is installed.
           $PIP install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple "numpy<2" "torch==2.2.2" torchvision
-          # Also ensure numpy<2 is pinned for the requirements.txt install below
           $PIP install --upgrade --force-reinstall "numpy<2"
           INSTALLED_SOMETHING=0
           if [[ -f requirements.txt ]]; then
@@ -380,9 +354,7 @@ jobs:
             exit 1
           fi
           $PIP install h5py scipy
-          # After requirements.txt, force numpy back to <2 (torch 2.2.2 has
-          # NumPy 1.x ABI; transitive deps in requirements.txt would
-          # otherwise leave numpy 2.x in place and crash at runtime).
+          # Force numpy<2 again (requirements.txt may have bumped it via transitive deps).
           $PIP install --upgrade --force-reinstall "numpy<2"
           TORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
           TORCH_REASON="no NVIDIA GPU detected"
@@ -403,8 +375,8 @@ jobs:
             fi
           fi
           echo "Installing PyTorch from ${TORCH_INDEX_URL} (${TORCH_REASON})."
-          # torch was pre-installed at the top of this step; no force-reinstall needed.
-          echo "Skipping torch force-reinstall (already installed at step head): $TORCH_INDEX_URL"
+          # torch pre-installed at step head; force-reinstall disabled to avoid 755MB redownload
+          echo "Skipping torch force-reinstall ($TORCH_INDEX_URL)"
 
       - name: Find and adapt dataset
         shell: bash
@@ -500,4 +472,4 @@ jobs:
             ${{ env.WAVESFM_OUTPUT_DIR }}/best.pth
             ${{ env.WAVESFM_OUTPUT_DIR }}/log.txt
           if-no-files-found: warn
-# committed at 2026-05-28T06:39:26.910514+00:00
+# committed at 2026-05-28T15:52:05.981554+00:00