From 8b33ee76e6573a5306e2813b1dd9390dbaee8df2 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 14:16:49 -0400 Subject: [PATCH 1/5] Find tests that are skipped in every configuration --- .github/workflows/ci.yml | 96 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292263f0c4..cd1a9fd78c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,6 +416,9 @@ jobs: name: Check job status if: always() runs-on: ubuntu-latest + permissions: + actions: read + contents: read needs: - should-skip - detect-changes @@ -426,6 +429,99 @@ jobs: - test-windows - doc steps: + - name: Report universally-skipped tests + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + REPO="${GITHUB_REPOSITORY}" + SHA="${GITHUB_SHA}" + WORKDIR=$(mktemp -d) + trap 'rm -rf "${WORKDIR}"' EXIT + + # Fetch all workflow runs associated with this commit (up to 100) + all_runs=$(gh api \ + "repos/${REPO}/actions/runs?head_sha=${SHA}&per_page=100" \ + --jq '.workflow_runs[]') + + # Identify child runs by their workflow file path; test-wheel-linux.yml + # covers both linux-64 and linux-aarch64 (two separate runs). + TEST_WORKFLOW_PATHS=( + ".github/workflows/test-sdist-linux.yml" + ".github/workflows/test-sdist-windows.yml" + ".github/workflows/test-wheel-linux.yml" + ".github/workflows/test-wheel-windows.yml" + ) + + mkdir -p "${WORKDIR}/configs" + config_index=0 + + for wf_path in "${TEST_WORKFLOW_PATHS[@]}"; do + run_ids=$(echo "${all_runs}" | jq -r \ + --arg p "${wf_path}" \ + 'select(.path == $p) | .id') + for run_id in ${run_ids}; do + run_name=$(echo "${all_runs}" | jq -r \ + --argjson id "${run_id}" 'select(.id == $id) | .name' | head -1) + echo "Fetching logs for: ${run_name} (id=${run_id})" + config_dir="${WORKDIR}/configs/${config_index}" + mkdir -p "${config_dir}" + + if gh api "repos/${REPO}/actions/runs/${run_id}/logs" \ + > "${config_dir}/logs.zip" 2>/dev/null; then + unzip -q "${config_dir}/logs.zip" -d "${config_dir}/logs" 2>/dev/null || true + # Extract test IDs from lines matching the pytest SKIPPED output format + grep -rh ' SKIPPED' "${config_dir}/logs/" 2>/dev/null \ + | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ + | sed 's/ SKIPPED$//' \ + | sort -u > "${config_dir}/skipped.txt" || true + echo " -> $(wc -l < "${config_dir}/skipped.txt") skipped tests" + config_index=$((config_index + 1)) + else + echo " -> could not download logs (run may be skipped or unavailable)" >&2 + rm -rf "${config_dir}" + fi + done + done + + { + echo "## Universally-skipped tests" + echo "" + } >> "${GITHUB_STEP_SUMMARY}" + + if [[ ${config_index} -eq 0 ]]; then + echo "_No test run logs found._" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + # Start the intersection from the first config and progressively + # narrow it down to only tests that appear in every config's list. + cp "${WORKDIR}/configs/0/skipped.txt" "${WORKDIR}/intersection.txt" + for i in $(seq 1 $((config_index - 1))); do + comm -12 \ + "${WORKDIR}/intersection.txt" \ + "${WORKDIR}/configs/${i}/skipped.txt" \ + > "${WORKDIR}/intersection_new.txt" + mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt" + done + + count=$(wc -l < "${WORKDIR}/intersection.txt") + { + echo "Tests skipped across all ${config_index} test configuration(s):" + echo "" + if [[ ${count} -eq 0 ]]; then + echo "_No tests were skipped in all configurations._" + else + echo "| Test |" + echo "| --- |" + while IFS= read -r test; do + [[ -z "${test}" ]] && continue + echo "| \`${test}\` |" + done < "${WORKDIR}/intersection.txt" + fi + } >> "${GITHUB_STEP_SUMMARY}" + - name: Exit run: | # if any dependencies were cancelled or failed, that's a failure From 8e4bd69f86b9540a0f4bee8862d98dc4f83e0dfc Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 16:01:35 -0400 Subject: [PATCH 2/5] Second try --- .github/workflows/ci.yml | 121 +++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd1a9fd78c..19fe7d259c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -436,53 +436,59 @@ jobs: set -euo pipefail REPO="${GITHUB_REPOSITORY}" - SHA="${GITHUB_SHA}" WORKDIR=$(mktemp -d) trap 'rm -rf "${WORKDIR}"' EXIT - # Fetch all workflow runs associated with this commit (up to 100) - all_runs=$(gh api \ - "repos/${REPO}/actions/runs?head_sha=${SHA}&per_page=100" \ - --jq '.workflow_runs[]') - - # Identify child runs by their workflow file path; test-wheel-linux.yml - # covers both linux-64 and linux-aarch64 (two separate runs). - TEST_WORKFLOW_PATHS=( - ".github/workflows/test-sdist-linux.yml" - ".github/workflows/test-sdist-windows.yml" - ".github/workflows/test-wheel-linux.yml" - ".github/workflows/test-wheel-windows.yml" + mkdir -p "${WORKDIR}/configs" + + # Collect all jobs for this run and group matrix jobs into the five + # top-level test configurations from ci.yml. + jobs_json=$(gh api --paginate \ + "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ + | jq -s '[.[].jobs[]]') + + declare -A CONFIG_PATTERNS=( + [test-sdist-linux]='^Test sdist linux-64 / ' + [test-sdist-windows]='^Test sdist win-64 / ' + [test-linux-64]='^Test linux-64 / ' + [test-linux-aarch64]='^Test linux-aarch64 / ' + [test-windows]='^Test (win-64|windows) / ' ) - mkdir -p "${WORKDIR}/configs" - config_index=0 - - for wf_path in "${TEST_WORKFLOW_PATHS[@]}"; do - run_ids=$(echo "${all_runs}" | jq -r \ - --arg p "${wf_path}" \ - 'select(.path == $p) | .id') - for run_id in ${run_ids}; do - run_name=$(echo "${all_runs}" | jq -r \ - --argjson id "${run_id}" 'select(.id == $id) | .name' | head -1) - echo "Fetching logs for: ${run_name} (id=${run_id})" - config_dir="${WORKDIR}/configs/${config_index}" - mkdir -p "${config_dir}" - - if gh api "repos/${REPO}/actions/runs/${run_id}/logs" \ - > "${config_dir}/logs.zip" 2>/dev/null; then - unzip -q "${config_dir}/logs.zip" -d "${config_dir}/logs" 2>/dev/null || true - # Extract test IDs from lines matching the pytest SKIPPED output format - grep -rh ' SKIPPED' "${config_dir}/logs/" 2>/dev/null \ - | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ - | sed 's/ SKIPPED$//' \ - | sort -u > "${config_dir}/skipped.txt" || true - echo " -> $(wc -l < "${config_dir}/skipped.txt") skipped tests" - config_index=$((config_index + 1)) - else - echo " -> could not download logs (run may be skipped or unavailable)" >&2 - rm -rf "${config_dir}" - fi + configs=( + test-sdist-linux + test-sdist-windows + test-linux-64 + test-linux-aarch64 + test-windows + ) + + for cfg in "${configs[@]}"; do + cfg_dir="${WORKDIR}/configs/${cfg}" + mkdir -p "${cfg_dir}/logs" + job_ids=$(echo "${jobs_json}" | jq -r \ + --arg pat "${CONFIG_PATTERNS[$cfg]}" \ + '.[] | select(.name | test($pat)) | .id') + + if [[ -z "${job_ids}" ]]; then + echo "No matrix jobs found for ${cfg}" >&2 + : > "${cfg_dir}/skipped.txt" + continue + fi + + for job_id in ${job_ids}; do + logfile="${cfg_dir}/logs/${job_id}.log" + # gh run view handles log retrieval for a job ID reliably. + gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true done + + # Extract test IDs from lines matching pytest SKIPPED output. + grep -h ' SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ + | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ + | sed 's/ SKIPPED$//' \ + | sort -u > "${cfg_dir}/skipped.txt" || true + + echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests" done { @@ -490,25 +496,42 @@ jobs: echo "" } >> "${GITHUB_STEP_SUMMARY}" - if [[ ${config_index} -eq 0 ]]; then - echo "_No test run logs found._" >> "${GITHUB_STEP_SUMMARY}" + available_configs=0 + missing_configs=() + for cfg in "${configs[@]}"; do + if [[ -s "${WORKDIR}/configs/${cfg}/skipped.txt" || -n "$(ls -A "${WORKDIR}/configs/${cfg}/logs" 2>/dev/null || true)" ]]; then + available_configs=$((available_configs + 1)) + else + missing_configs+=("${cfg}") + fi + done + + if [[ ${available_configs} -eq 0 ]]; then + echo "_No test job logs found in this run._" >> "${GITHUB_STEP_SUMMARY}" exit 0 fi - # Start the intersection from the first config and progressively - # narrow it down to only tests that appear in every config's list. - cp "${WORKDIR}/configs/0/skipped.txt" "${WORKDIR}/intersection.txt" - for i in $(seq 1 $((config_index - 1))); do + if [[ ${#missing_configs[@]} -gt 0 ]]; then + { + echo "_Warning: missing logs for configuration(s): ${missing_configs[*]}_" + echo "" + } >> "${GITHUB_STEP_SUMMARY}" + fi + + # Start intersection from first config, then narrow with each of the + # remaining four configurations. + cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt" + for cfg in "${configs[@]:1}"; do comm -12 \ "${WORKDIR}/intersection.txt" \ - "${WORKDIR}/configs/${i}/skipped.txt" \ + "${WORKDIR}/configs/${cfg}/skipped.txt" \ > "${WORKDIR}/intersection_new.txt" mv "${WORKDIR}/intersection_new.txt" "${WORKDIR}/intersection.txt" done count=$(wc -l < "${WORKDIR}/intersection.txt") { - echo "Tests skipped across all ${config_index} test configuration(s):" + echo "Tests skipped across all five test configurations:" echo "" if [[ ${count} -eq 0 ]]; then echo "_No tests were skipped in all configurations._" From 942725cbec88f03c0aeaca390f9e5db5889e23ba Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 23 Apr 2026 17:02:59 -0400 Subject: [PATCH 3/5] Add a test that always skips to test this --- cuda_bindings/tests/test_cuda.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index e3eefb1fdd..0ea86a82c8 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -40,6 +40,11 @@ def callableBinary(name): return shutil.which(name) is not None +@pytest.mark.skipif(True, "Always skip!") +def test_always_skip(): + pass + + def test_cuda_memcpy(): # Get device From ebcbb8a1739c6d8f8532335a12750167baa87525 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 24 Apr 2026 10:34:31 -0400 Subject: [PATCH 4/5] Add reason --- cuda_bindings/tests/test_cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_bindings/tests/test_cuda.py b/cuda_bindings/tests/test_cuda.py index 0ea86a82c8..86b359ce04 100644 --- a/cuda_bindings/tests/test_cuda.py +++ b/cuda_bindings/tests/test_cuda.py @@ -40,7 +40,7 @@ def callableBinary(name): return shutil.which(name) is not None -@pytest.mark.skipif(True, "Always skip!") +@pytest.mark.skipif(True, reason="Always skip!") def test_always_skip(): pass From 63ca53a83f222f5e4d6b881f3f1997189b10d518 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 24 Apr 2026 15:04:57 -0400 Subject: [PATCH 5/5] Hopefully working now --- .github/workflows/ci.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19fe7d259c..d5f8024097 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -441,23 +441,19 @@ jobs: mkdir -p "${WORKDIR}/configs" - # Collect all jobs for this run and group matrix jobs into the five - # top-level test configurations from ci.yml. + # Collect all jobs for this run and group matrix jobs into wheel + # test configurations from ci.yml. jobs_json=$(gh api --paginate \ "repos/${REPO}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ | jq -s '[.[].jobs[]]') declare -A CONFIG_PATTERNS=( - [test-sdist-linux]='^Test sdist linux-64 / ' - [test-sdist-windows]='^Test sdist win-64 / ' [test-linux-64]='^Test linux-64 / ' [test-linux-aarch64]='^Test linux-aarch64 / ' [test-windows]='^Test (win-64|windows) / ' ) configs=( - test-sdist-linux - test-sdist-windows test-linux-64 test-linux-aarch64 test-windows @@ -478,14 +474,18 @@ jobs: for job_id in ${job_ids}; do logfile="${cfg_dir}/logs/${job_id}.log" - # gh run view handles log retrieval for a job ID reliably. - gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true + # Prefer the job log API; fall back to gh run view if needed. + if ! gh api "repos/${REPO}/actions/jobs/${job_id}/logs" > "${logfile}" 2>/dev/null; then + gh run view "${GITHUB_RUN_ID}" --job "${job_id}" --log > "${logfile}" || true + fi done - # Extract test IDs from lines matching pytest SKIPPED output. - grep -h ' SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ - | grep -oE '[^[:space:]]+\.py::[^[:space:]]+ SKIPPED' \ - | sed 's/ SKIPPED$//' \ + # Extract pytest node IDs from SKIPPED lines. This tolerates + # timestamped logs, ANSI escapes, and trailing skip reasons. + grep -h 'SKIPPED' "${cfg_dir}/logs/"*.log 2>/dev/null \ + | sed -E 's/\x1B\[[0-9;]*[[:alpha:]]//g' \ + | sed 's#\\#/#g' \ + | sed -nE 's#.*(tests/[[:graph:]]+\.py::[^[:space:]]+).*SKIPPED.*#\1#p' \ | sort -u > "${cfg_dir}/skipped.txt" || true echo "${cfg}: $(wc -l < "${cfg_dir}/skipped.txt") skipped tests" @@ -518,8 +518,7 @@ jobs: } >> "${GITHUB_STEP_SUMMARY}" fi - # Start intersection from first config, then narrow with each of the - # remaining four configurations. + # Start intersection from the first wheel configuration, then narrow. cp "${WORKDIR}/configs/${configs[0]}/skipped.txt" "${WORKDIR}/intersection.txt" for cfg in "${configs[@]:1}"; do comm -12 \ @@ -531,7 +530,7 @@ jobs: count=$(wc -l < "${WORKDIR}/intersection.txt") { - echo "Tests skipped across all five test configurations:" + echo "Tests skipped across wheel test configurations (${#configs[@]}):" echo "" if [[ ${count} -eq 0 ]]; then echo "_No tests were skipped in all configurations._"