deletdd files, updated workflow

.gitattributes

@@ -1,4 +0,0 @@
-*.npy filter=lfs diff=lfs merge=lfs -text
-recordings filter=lfs diff=lfs merge=lfs -text
-recordings/** filter=lfs diff=lfs merge=lfs -text
-*.whl filter=lfs diff=lfs merge=lfs -text

.riahub/workflows/workflow.yaml

@@ -37,10 +37,22 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: 1. Build HDF5 Dataset
+      - name: 1. Generate Recordings
+        run: |
+         mkdir -p data/recordings
+         PYTHONPATH=. python scripts/dataset_building/data_gen.py
+         echo "recordings produced successfully"
+        
+      - name: Upload Recordings
+        uses: actions/upload-artifact@v3
+        with:
+          name: recordings
+          path: data/recordings/**
+
+      - name: 2. Build HDF5 Dataset
         run: |
          mkdir -p data/dataset
-         PYTHONPATH=. python data/scripts/produce_dataset.py
+         PYTHONPATH=. python scripts/dataset_building/produce_dataset.py
          echo "datasets produced successfully"
         shell: bash
       
@@ -51,12 +63,12 @@ jobs:
           path: data/dataset/**
 
 
-      - name: 2. Train Model
+      - name: 3. Train Model
         env:
             NO_NNPACK: 1
             PYTORCH_NO_NNPACK: 1
         run: |
-          PYTHONPATH=. python data/training/train.py 
+          PYTHONPATH=. python scripts/training/train.py 
           echo "training model"
 
       - name: Upload Checkpoints
@@ -65,7 +77,7 @@ jobs:
           name: ria-checkpoints
           path: checkpoint_files/inference_recognition_model.ckpt
 
-      - name: 3. Convert to ONNX file
+      - name: 4. Convert to ONNX file
         run: |
           MKL_DISABLE_FAST_MM=1 PYTHONPATH=. python onnx_scripts/convert_to_onnx.py
           echo "building inference app"
@@ -76,7 +88,7 @@ jobs:
           name: ria-demo-onnx
           path: onnx_files/inference_recognition_model.onnx
 
-      - name: 4. Profile ONNX model
+      - name: 5. Profile ONNX model
         run: |
           PYTHONPATH=. python onnx_scripts/profile_onnx.py
       
@@ -86,7 +98,7 @@ jobs:
           name: profile-data          
           path: '**/onnxruntime_profile_*.json'
       
-      - name: 5. Convert to ORT file
+      - name: 6. Convert to ORT file
         run: |
           python -m onnxruntime.tools.convert_onnx_models_to_ort \
              /workspace/qoherent/modrec-workflow/onnx_files/inference_recognition_model.onnx \

README.md

@@ -70,5 +70,4 @@ Coding Guidelines:
   Use descriptive commit messages and reference issue numbers when relevant.
 Contributing
   All contributions must be reviewed via pull requests.
-  Run all tests and ensure code passes lint checks before submission.
+  Run all tests and ensure code passes lint checks before submission.
-  See CONTRIBUTING.md for more details (if present).

checkpoint_files/__init__.py

@@ -1,6 +0,0 @@
-import os
-
-CHECKPOINTS_DIR = os.path.dirname(os.path.abspath(__file__))
-
-if __name__ == "__main__":
-    print(CHECKPOINTS_DIR)

conf/app.yaml

@@ -3,24 +3,43 @@ general:
   run_mode: prod
 
 dataset:
+  #where to read the recordings from to produce the data set
   input_dir: data/recordings
+
+  #number of slices you want to split each recording into
   num_slices: 8
+
+  #training/val split between the 2 data sets
   train_split: 0.8
   val_split : 0.2
+
+  #used to initialize a random number generator.
   seed: 25
+
+  #multiple modulations to contain in the dataset
   modulation_types: [bpsk, qpsk, qam16, qam64]
+
+  #where to output the datasets
   output_dir: data/dataset
 
 training:
+  #number of training samples being processed together before model updates its weights
   batch_size: 256
+  
+  #number of passes through the data set during the training process
   epochs: 5
+
+  #how much the weights update during training after every batch
+  #suggested range for fine-tuning: (1e-6, 1e-4)
   learning_rate: 1e-4
-  checkpoint_dir: checkpoint_files
+
-  checkpoint_filename: inference_recognition_model
   use_gpu: true
 
 inference:
+  #num classes to classify on
   num_classes: 4
+
+  #output of the onnx model
   onnx_model_filename: inference_recognition_model
 
 app: