diff --git a/.github/workflows/stress-tests.yml b/.github/workflows/stress-tests.yml index 6592f2510..4e464a137 100644 --- a/.github/workflows/stress-tests.yml +++ b/.github/workflows/stress-tests.yml @@ -49,3 +49,103 @@ jobs: logs/ TestResults/StressTests + parallel_unittests_stress: + name: Parallel Unit Tests Stress (3 iterations) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + + timeout-minutes: 90 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup .NET Core + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + dotnet-quality: 'ga' + + - name: Install dependencies + run: dotnet restore + + - name: Build UnitTestsParallelizable + run: dotnet build Tests/UnitTestsParallelizable --configuration Debug --no-restore + + - name: Disable Windows Defender (Windows only) + if: runner.os == 'Windows' + shell: powershell + run: | + Add-MpPreference -ExclusionPath "${{ github.workspace }}" + Add-MpPreference -ExclusionProcess "dotnet.exe" + Add-MpPreference -ExclusionProcess "testhost.exe" + Add-MpPreference -ExclusionProcess "VSTest.Console.exe" + + - name: Set VSTEST_DUMP_PATH + shell: bash + run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV + + - name: Run UnitTestsParallelizable (3 iterations with varying parallelization) + shell: bash + run: | + # Run tests 3 times with different parallelization settings to expose concurrency issues + # Run 1: Default parallelization (2x) - standard test execution + # Run 2: Maximum parallelization (unlimited) - stress test with high concurrency + # Run 3: Single-threaded execution (1) - deterministic execution to expose ordering issues + for RUN in {1..3}; do + echo "============================================" + echo "Starting test run $RUN of 3" + echo "============================================" + + # Use a combination of run number and timestamp to create different execution patterns + SEED=$((1000 + $RUN + $(date +%s) % 1000)) + echo "Using randomization seed: $SEED" + + # Vary the xUnit parallelization based on run number to expose race conditions + if [ $RUN -eq 1 ]; then + XUNIT_MAX_PARALLEL_THREADS="2x" + echo "Run $RUN: Using default parallelization (2x)" + elif [ $RUN -eq 2 ]; then + XUNIT_MAX_PARALLEL_THREADS="unlimited" + echo "Run $RUN: Using maximum parallelization (unlimited)" + else + XUNIT_MAX_PARALLEL_THREADS="1" + echo "Run $RUN: Using single-threaded execution" + fi + + dotnet test Tests/UnitTestsParallelizable \ + --no-build \ + --verbosity normal \ + --settings Tests/UnitTestsParallelizable/runsettings.xml \ + --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \ + --blame \ + --blame-crash \ + --blame-hang \ + --blame-hang-timeout 60s \ + --blame-crash-collect-always \ + -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS} + + if [ $? -ne 0 ]; then + echo "ERROR: Test run $RUN failed!" + exit 1 + fi + + echo "Test run $RUN completed successfully" + echo "" + done + + echo "============================================" + echo "All 3 test runs completed successfully!" + echo "============================================" + + - name: Upload UnitTestsParallelizable Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: parallel_unittests_stress-logs-${{ runner.os }} + path: | + logs/UnitTestsParallelizable/ + TestResults/ + diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5200800cf..7b936fc5c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -154,80 +154,48 @@ jobs: shell: bash run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV - - name: Run UnitTestsParallelizable (10 iterations with varying parallelization) + - name: Run UnitTestsParallelizable shell: bash run: | - # Run tests 3 times with different parallelization settings to expose concurrency issues - for RUN in {1..3}; do - echo "============================================" - echo "Starting test run $RUN of 3" - echo "============================================" - - # Use a combination of run number and timestamp to create different execution patterns - SEED=$((1000 + $RUN + $(date +%s) % 1000)) - echo "Using randomization seed: $SEED" - - # Vary the xUnit parallelization based on run number to expose race conditions - # Runs 1-3: Default parallelization (2x CPU cores) - # Runs 4-6: Max parallelization (unlimited) - # Runs 7-9: Single threaded (1) - # Run 10: Random (1-4 threads) - if [ $RUN -le 3 ]; then - XUNIT_MAX_PARALLEL_THREADS="2x" - echo "Run $RUN: Using default parallelization (2x)" - elif [ $RUN -le 6 ]; then - XUNIT_MAX_PARALLEL_THREADS="unlimited" - echo "Run $RUN: Using maximum parallelization (unlimited)" - elif [ $RUN -le 9 ]; then - XUNIT_MAX_PARALLEL_THREADS="1" - echo "Run $RUN: Using single-threaded execution" - else - # Random parallelization based on seed - PROC_COUNT=$(( ($SEED % 4) + 1 )) - XUNIT_MAX_PARALLEL_THREADS="$PROC_COUNT" - echo "Run $RUN: Using random parallelization with $PROC_COUNT threads" - fi - - # Run tests with or without coverage based on OS and run number - if [ "${{ runner.os }}" == "Linux" ] && [ $RUN -eq 1 ]; then - echo "Run $RUN: Running with coverage collection" - dotnet test Tests/UnitTestsParallelizable \ - --no-build \ - --verbosity normal \ - --collect:"XPlat Code Coverage" \ - --settings Tests/UnitTests/runsettings.coverage.xml \ - --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \ - --blame \ - --blame-crash \ - --blame-hang \ - --blame-hang-timeout 60s \ - --blame-crash-collect-always \ - -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS} - else - dotnet test Tests/UnitTestsParallelizable \ - --no-build \ - --verbosity normal \ - --settings Tests/UnitTestsParallelizable/runsettings.xml \ - --diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \ - --blame \ - --blame-crash \ - --blame-hang \ - --blame-hang-timeout 60s \ - --blame-crash-collect-always \ - -- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS} - fi - - if [ $? -ne 0 ]; then - echo "ERROR: Test run $RUN failed!" - exit 1 - fi - - echo "Test run $RUN completed successfully" - echo "" - done + # Run tests once for regular workflow runs (stress tests will run multiple iterations) + echo "============================================" + echo "Starting parallel unit tests" + echo "============================================" + + # Use default parallelization (2x CPU cores) + echo "Using default parallelization (2x)" + + # Run tests with or without coverage based on OS + if [ "${{ runner.os }}" == "Linux" ]; then + echo "Running with coverage collection on Linux" + dotnet test Tests/UnitTestsParallelizable \ + --no-build \ + --verbosity normal \ + --collect:"XPlat Code Coverage" \ + --settings Tests/UnitTests/runsettings.coverage.xml \ + --diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \ + --blame \ + --blame-crash \ + --blame-hang \ + --blame-hang-timeout 60s \ + --blame-crash-collect-always \ + -- xUnit.MaxParallelThreads=2x + else + dotnet test Tests/UnitTestsParallelizable \ + --no-build \ + --verbosity normal \ + --settings Tests/UnitTestsParallelizable/runsettings.xml \ + --diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \ + --blame \ + --blame-crash \ + --blame-hang \ + --blame-hang-timeout 60s \ + --blame-crash-collect-always \ + -- xUnit.MaxParallelThreads=2x + fi echo "============================================" - echo "All 10 test runs completed successfully!" + echo "Parallel unit tests completed successfully!" echo "============================================" - name: Upload UnitTestsParallelizable Logs