Dual-canary: ICC-DEMO training run (icc28-train_v1.0.0.h5)
Some checks failed
WavesFM Fine-Tuning / WavesFM-Training (push) Failing after 39s

This commit is contained in:
P Roman Pope 2026-05-28 07:36:31 -04:00
parent 6da3ce6204
commit 548a00ee72

View File

@ -15,11 +15,11 @@ permissions:
jobs: jobs:
WavesFM-Training: WavesFM-Training:
runs-on: "ubuntu-24.04" runs-on: "ubuntu-latest"
env: env:
WAVESFM_TASK: "rml" WAVESFM_TASK: "rml"
WAVESFM_EPOCHS: "10" WAVESFM_EPOCHS: "1"
WAVESFM_BATCH_SIZE: "2048" WAVESFM_BATCH_SIZE: "8"
WAVESFM_OUTPUT_DIR: "/opt/wavesfm/output" WAVESFM_OUTPUT_DIR: "/opt/wavesfm/output"
# Single source of truth for the cloned WavesFM repo location. # Single source of truth for the cloned WavesFM repo location.
# Referenced as ${{ env.WAVESFM_REPO_DIR }} in steps. To relocate # Referenced as ${{ env.WAVESFM_REPO_DIR }} in steps. To relocate
@ -44,6 +44,7 @@ jobs:
echo "No NVIDIA GPU available." echo "No NVIDIA GPU available."
fi fi
- name: "Download Model (qoherent/wavesfm-base/wavesfm-v1p0.pth)" - name: "Download Model (qoherent/wavesfm-base/wavesfm-v1p0.pth)"
shell: bash shell: bash
timeout-minutes: 4 timeout-minutes: 4
@ -180,7 +181,8 @@ jobs:
fi fi
exit 1 exit 1
fi fi
- name: Checkout Training Dataset
- name: "Checkout Dataset (qoherent/icc-demo/datasets/icc28-train_v1.0.0.h5)"
shell: bash shell: bash
timeout-minutes: 10 timeout-minutes: 10
env: env:
@ -210,8 +212,16 @@ jobs:
AUTH_HEADER="" AUTH_HEADER=""
if [[ -n "${RIAHUB_USER:-}" && -n "${RIAHUB_TOKEN:-}" ]]; then if [[ -n "${RIAHUB_USER:-}" && -n "${RIAHUB_TOKEN:-}" ]]; then
AUTH_HEADER=$(printf 'Authorization: basic %s' "$(printf '%s:%s' "$RIAHUB_USER" "$RIAHUB_TOKEN" | base64 | tr -d '\n')") AUTH_HEADER=$(printf 'Authorization: basic %s' \
"$(printf '%s:%s' "$RIAHUB_USER" "$RIAHUB_TOKEN" | base64 | tr -d '\n')")
fi fi
# ``sudo env GIT_TERMINAL_PROMPT=0`` propagates the env var across
# sudo's default ``env_reset`` boundary; a bare ``sudo git`` would
# see an empty env on most distros' default sudoers, so the
# step-level ``env:`` block's GIT_TERMINAL_PROMPT=0 would NOT
# actually reach git child processes. Without it, git falls back
# to opening ``/dev/tty`` (the PTY allocated by act_runner) and
# prompting for credentials on a 401, hanging until timeout.
git_auth() { git_auth() {
if [[ -n "$AUTH_HEADER" ]]; then if [[ -n "$AUTH_HEADER" ]]; then
sudo env GIT_TERMINAL_PROMPT=0 git -c "http.extraheader=$AUTH_HEADER" "$@" sudo env GIT_TERMINAL_PROMPT=0 git -c "http.extraheader=$AUTH_HEADER" "$@"
@ -219,8 +229,9 @@ jobs:
sudo env GIT_TERMINAL_PROMPT=0 git "$@" sudo env GIT_TERMINAL_PROMPT=0 git "$@"
fi fi
} }
REPO_PATH='/qoherent/icc-28.git'
DEST_ROOT='/opt/qmb/riahub/dataset/qoherent/icc-28/main' REPO_PATH='/qoherent/icc-demo.git'
DEST_ROOT='/opt/qmb/riahub/dataset/qoherent/icc-demo/ef0a0e430f8e5019dec52794fccab958b3a3c2b7'
sudo mkdir -p "$(dirname "$DEST_ROOT")" sudo mkdir -p "$(dirname "$DEST_ROOT")"
if ! command -v git-lfs >/dev/null 2>&1; then if ! command -v git-lfs >/dev/null 2>&1; then
sudo apt-get update -y sudo apt-get update -y
@ -236,13 +247,28 @@ jobs:
sudo mkdir -p "$DEST_ROOT" sudo mkdir -p "$DEST_ROOT"
sudo git -C "$DEST_ROOT" init || continue sudo git -C "$DEST_ROOT" init || continue
sudo git -C "$DEST_ROOT" remote add origin "$REPO_URL" || continue sudo git -C "$DEST_ROOT" remote add origin "$REPO_URL" || continue
# See ``_render_model_checkout`` for the rationale on skipping
# ``git lfs install --local`` — short version: the smudge
# filter it would register tries its own credential lookup
# during ``git checkout FETCH_HEAD`` and hangs forever on
# /dev/tty when the repo is internal/private. We rely on
# the explicit ``git lfs fetch`` (with auth) +
# ``git lfs checkout`` (local) pair below instead.
sudo git -C "$DEST_ROOT" sparse-checkout init --no-cone || continue sudo git -C "$DEST_ROOT" sparse-checkout init --no-cone || continue
sudo git -C "$DEST_ROOT" sparse-checkout set --no-cone -- \ sudo git -C "$DEST_ROOT" sparse-checkout set --no-cone -- \
'datasets/icc28-train_v1.0.0.h5' || continue 'datasets/icc28-train_v1.0.0.h5' || continue
if ! git_auth -C "$DEST_ROOT" fetch --depth=1 origin '1d9083f05d0538110f09e710865b078eba30964b'; then if ! git_auth -C "$DEST_ROOT" fetch --depth=1 origin 'ef0a0e430f8e5019dec52794fccab958b3a3c2b7'; then
continue continue
fi fi
if ! sudo env GIT_TERMINAL_PROMPT=0 GIT_LFS_SKIP_SMUDGE=1 git -C "$DEST_ROOT" -c advice.detachedHead=false checkout FETCH_HEAD; then # See ``_render_model_checkout`` for the rationale on
# ``GIT_LFS_SKIP_SMUDGE=1`` — short version: the runner has
# the LFS smudge filter installed system-wide
# (``/etc/gitconfig``), so checkout fires it and the filter's
# credential helper hangs on /dev/tty for internal repos.
# Skipping smudge here lets the explicit ``git lfs fetch``
# below handle materialization with proper auth.
if ! sudo env GIT_TERMINAL_PROMPT=0 GIT_LFS_SKIP_SMUDGE=1 \
git -C "$DEST_ROOT" -c advice.detachedHead=false checkout FETCH_HEAD; then
continue continue
fi fi
if ! git_auth -C "$DEST_ROOT" lfs fetch origin --include='datasets/icc28-train_v1.0.0.h5' --exclude=""; then if ! git_auth -C "$DEST_ROOT" lfs fetch origin --include='datasets/icc28-train_v1.0.0.h5' --exclude=""; then
@ -268,7 +294,7 @@ jobs:
if [[ "$MATERIALIZED" -ne 1 ]]; then if [[ "$MATERIALIZED" -ne 1 ]]; then
echo "Failed to materialize dataset using base URL candidates derived from: $BASE_URL_SOURCE" >&2 echo "Failed to materialize dataset using base URL candidates derived from: $BASE_URL_SOURCE" >&2
if [[ -z "$AUTH_HEADER" ]]; then if [[ -z "$AUTH_HEADER" ]]; then
echo " (no credentials configured: set QMBDEMO_USER and QMBDEMO_TOKEN repo secrets for internal/private repos)" >&2 echo " (set QMBDEMO_USER+QMBDEMO_TOKEN repo secrets for internal/private repos)" >&2
fi fi
exit 1 exit 1
fi fi
@ -472,4 +498,4 @@ jobs:
${{ env.WAVESFM_OUTPUT_DIR }}/best.pth ${{ env.WAVESFM_OUTPUT_DIR }}/best.pth
${{ env.WAVESFM_OUTPUT_DIR }}/log.txt ${{ env.WAVESFM_OUTPUT_DIR }}/log.txt
if-no-files-found: warn if-no-files-found: warn
# committed at 2026-05-28T11:19:35.092486+00:00 # committed at 2026-05-28T11:36:29.397013+00:00