Move parallel test repeat logic to stress tests (#4455)

* Initial plan * Move parallel test repeat logic to stress tests Co-authored-by: tig <585482+tig@users.noreply.github.com> * Simplify unit tests workflow and improve comments Co-authored-by: tig <585482+tig@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: tig <585482+tig@users.noreply.github.com>
2025-12-26 15:57:56 +01:00 · 2025-12-06 11:58:13 -07:00
parent 7a8b6e4465
commit 0eafb5947e
2 changed files with 138 additions and 70 deletions
--- a/.github/workflows/stress-tests.yml
+++ b/.github/workflows/stress-tests.yml
@@ -49,3 +49,103 @@ jobs:
            logs/
            TestResults/StressTests

+  parallel_unittests_stress:
+    name: Parallel Unit Tests Stress (3 iterations)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.x
+          dotnet-quality: 'ga'
+
+      - name: Install dependencies
+        run: dotnet restore
+
+      - name: Build UnitTestsParallelizable
+        run: dotnet build Tests/UnitTestsParallelizable --configuration Debug --no-restore
+
+      - name: Disable Windows Defender (Windows only)
+        if: runner.os == 'Windows'
+        shell: powershell
+        run: |
+          Add-MpPreference -ExclusionPath "${{ github.workspace }}"
+          Add-MpPreference -ExclusionProcess "dotnet.exe"
+          Add-MpPreference -ExclusionProcess "testhost.exe"
+          Add-MpPreference -ExclusionProcess "VSTest.Console.exe"
+
+      - name: Set VSTEST_DUMP_PATH
+        shell: bash
+        run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV
+
+      - name: Run UnitTestsParallelizable (3 iterations with varying parallelization)
+        shell: bash
+        run: |
+          # Run tests 3 times with different parallelization settings to expose concurrency issues
+          # Run 1: Default parallelization (2x) - standard test execution
+          # Run 2: Maximum parallelization (unlimited) - stress test with high concurrency
+          # Run 3: Single-threaded execution (1) - deterministic execution to expose ordering issues
+          for RUN in {1..3}; do
+            echo "============================================"
+            echo "Starting test run $RUN of 3"
+            echo "============================================"
+            
+            # Use a combination of run number and timestamp to create different execution patterns
+            SEED=$((1000 + $RUN + $(date +%s) % 1000))
+            echo "Using randomization seed: $SEED"
+            
+            # Vary the xUnit parallelization based on run number to expose race conditions
+            if [ $RUN -eq 1 ]; then
+              XUNIT_MAX_PARALLEL_THREADS="2x"
+              echo "Run $RUN: Using default parallelization (2x)"
+            elif [ $RUN -eq 2 ]; then
+              XUNIT_MAX_PARALLEL_THREADS="unlimited"
+              echo "Run $RUN: Using maximum parallelization (unlimited)"
+            else
+              XUNIT_MAX_PARALLEL_THREADS="1"
+              echo "Run $RUN: Using single-threaded execution"
+            fi
+            
+            dotnet test Tests/UnitTestsParallelizable \
+              --no-build \
+              --verbosity normal \
+              --settings Tests/UnitTestsParallelizable/runsettings.xml \
+              --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
+              --blame \
+              --blame-crash \
+              --blame-hang \
+              --blame-hang-timeout 60s \
+              --blame-crash-collect-always \
+              -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
+            
+            if [ $? -ne 0 ]; then
+              echo "ERROR: Test run $RUN failed!"
+              exit 1
+            fi
+            
+            echo "Test run $RUN completed successfully"
+            echo ""
+          done
+          
+          echo "============================================"
+          echo "All 3 test runs completed successfully!"
+          echo "============================================"
+
+      - name: Upload UnitTestsParallelizable Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: parallel_unittests_stress-logs-${{ runner.os }}
+          path: |
+            logs/UnitTestsParallelizable/
+            TestResults/
+
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -154,80 +154,48 @@ jobs:
      shell: bash
      run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV

-    - name: Run UnitTestsParallelizable (10 iterations with varying parallelization)
+    - name: Run UnitTestsParallelizable
      shell: bash
      run: |
-        # Run tests 3 times with different parallelization settings to expose concurrency issues
-        for RUN in {1..3}; do
-          echo "============================================"
-          echo "Starting test run $RUN of 3"
-          echo "============================================"
-          
-          # Use a combination of run number and timestamp to create different execution patterns
-          SEED=$((1000 + $RUN + $(date +%s) % 1000))
-          echo "Using randomization seed: $SEED"
-          
-          # Vary the xUnit parallelization based on run number to expose race conditions
-          # Runs 1-3: Default parallelization (2x CPU cores)
-          # Runs 4-6: Max parallelization (unlimited)
-          # Runs 7-9: Single threaded (1)
-          # Run 10: Random (1-4 threads)
-          if [ $RUN -le 3 ]; then
-            XUNIT_MAX_PARALLEL_THREADS="2x"
-            echo "Run $RUN: Using default parallelization (2x)"
-          elif [ $RUN -le 6 ]; then
-            XUNIT_MAX_PARALLEL_THREADS="unlimited"
-            echo "Run $RUN: Using maximum parallelization (unlimited)"
-          elif [ $RUN -le 9 ]; then
-            XUNIT_MAX_PARALLEL_THREADS="1"
-            echo "Run $RUN: Using single-threaded execution"
-          else
-            # Random parallelization based on seed
-            PROC_COUNT=$(( ($SEED % 4) + 1 ))
-            XUNIT_MAX_PARALLEL_THREADS="$PROC_COUNT"
-            echo "Run $RUN: Using random parallelization with $PROC_COUNT threads"
-          fi
-          
-          # Run tests with or without coverage based on OS and run number
-          if [ "${{ runner.os }}" == "Linux" ] && [ $RUN -eq 1 ]; then
-            echo "Run $RUN: Running with coverage collection"
-            dotnet test Tests/UnitTestsParallelizable \
-              --no-build \
-              --verbosity normal \
-              --collect:"XPlat Code Coverage" \
-              --settings Tests/UnitTests/runsettings.coverage.xml \
-              --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
-              --blame \
-              --blame-crash \
-              --blame-hang \
-              --blame-hang-timeout 60s \
-              --blame-crash-collect-always \
-              -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
-          else
-            dotnet test Tests/UnitTestsParallelizable \
-              --no-build \
-              --verbosity normal \
-              --settings Tests/UnitTestsParallelizable/runsettings.xml \
-              --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
-              --blame \
-              --blame-crash \
-              --blame-hang \
-              --blame-hang-timeout 60s \
-              --blame-crash-collect-always \
-              -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
-          fi
-          
-          if [ $? -ne 0 ]; then
-            echo "ERROR: Test run $RUN failed!"
-            exit 1
-          fi
-          
-          echo "Test run $RUN completed successfully"
-          echo ""
-        done
+        # Run tests once for regular workflow runs (stress tests will run multiple iterations)
+        echo "============================================"
+        echo "Starting parallel unit tests"
+        echo "============================================"
+        
+        # Use default parallelization (2x CPU cores)
+        echo "Using default parallelization (2x)"
+        
+        # Run tests with or without coverage based on OS
+        if [ "${{ runner.os }}" == "Linux" ]; then
+          echo "Running with coverage collection on Linux"
+          dotnet test Tests/UnitTestsParallelizable \
+            --no-build \
+            --verbosity normal \
+            --collect:"XPlat Code Coverage" \
+            --settings Tests/UnitTests/runsettings.coverage.xml \
+            --diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \
+            --blame \
+            --blame-crash \
+            --blame-hang \
+            --blame-hang-timeout 60s \
+            --blame-crash-collect-always \
+            -- xUnit.MaxParallelThreads=2x
+        else
+          dotnet test Tests/UnitTestsParallelizable \
+            --no-build \
+            --verbosity normal \
+            --settings Tests/UnitTestsParallelizable/runsettings.xml \
+            --diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \
+            --blame \
+            --blame-crash \
+            --blame-hang \
+            --blame-hang-timeout 60s \
+            --blame-crash-collect-always \
+            -- xUnit.MaxParallelThreads=2x
+        fi
        
        echo "============================================"
-        echo "All 10 test runs completed successfully!"
+        echo "Parallel unit tests completed successfully!"
        echo "============================================"

    - name: Upload UnitTestsParallelizable Logs