trying something out

.circleci/create_circleci_config.py

@@ -61,9 +61,15 @@ class EmptyJob:
     def to_dict(self):
         steps = [{"run": 'ls -la'}]
         if self.job_name == "collection_job":
+            # Export the PR number once (if we have one) so the failure summary can post a GitHub comment.
             steps.extend(
                 [
                     "checkout",
+                    {
+                        "run": (
+                            'echo "export PR_NUMBER=$(python utils/extract_pr_number_from_circleci.py)" >> $BASH_ENV'
+                        )
+                    },
                     {"run": "pip install requests || true"},
                     {"run": """while [[ $(curl --location --request GET "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" --header "Circle-Token: $CCI_TOKEN"| jq -r '.items[]|select(.name != "collection_job")|.status' | grep -c "running") -gt 0 ]]; do sleep 5; done || true"""},
                     {"run": 'python utils/process_circleci_workflow_test_reports.py --workflow_id $CIRCLE_WORKFLOW_ID || true'},

.github/workflows/circleci-failure-summary-comment.yml

@@ -0,0 +1,117 @@
+name: CircleCI Failure Summary Comment
+# Requires repository secrets:
+# - CI_ARTIFACT_TOKEN: API token with permission to query CircleCI pipelines (same value used by CircleCI contexts)
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  comment:
+    runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: write
+    env:
+      TARGET_BRANCH: ${{ github.event.pull_request.head.ref }}
+      TARGET_SHA: ${{ github.event.pull_request.head.sha }}
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+      CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: python -m pip install requests huggingface_hub
+
+      - name: Wait for CircleCI check suite completion
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          github_repository: ${{ github.repository }}
+        run: |
+          echo "Waiting for CircleCI check suite to complete..."
+          end=$((SECONDS+1800))
+          while [ $SECONDS -lt $end ]; do
+            suite_json=$(gh api "repos/${github_repository}/commits/${COMMIT_SHA}/check-suites" --jq '.check_suites[] | select(.app.slug=="circleci-checks")')
+            if [ -z "$suite_json" ]; then
+              echo "CircleCI check suite not found yet, retrying..."
+            else
+              status=$(echo "$suite_json" | jq -r '.status')
+              conclusion=$(echo "$suite_json" | jq -r '.conclusion // empty')
+              echo "Current CircleCI check suite status: $status (conclusion: $conclusion)"
+              if [ "$status" = "completed" ] && [ -n "$conclusion" ]; then
+                break
+              fi
+            fi
+            sleep 20
+          done
+          if [ $SECONDS -ge $end ]; then
+            echo "Timed out waiting for CircleCI check suite."
+            exit 1
+          fi
+
+      - name: Find CircleCI workflow
+        id: circleci
+        env:
+          CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
+        run: |
+          WORKFLOW_ID=$(python scripts/find_circleci_workflow.py --branch "$TARGET_BRANCH" --sha "$TARGET_SHA")
+          echo "workflow_id=$WORKFLOW_ID" >> $GITHUB_OUTPUT
+
+      - name: Generate failure summary
+        env:
+          CIRCLE_TOKEN: ${{ secrets.CI_ARTIFACT_TOKEN }}
+        run: |
+          python utils/process_circleci_workflow_test_reports.py --workflow_id "${{ steps.circleci.outputs.workflow_id }}"
+
+      - name: Upload summaries to Hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_CI_WRITE_TOKEN }}
+          CIRCLECI_RESULTS_DATASET_ID: "transformers-community/circleci-test-results"
+        run: |
+          python utils/upload_circleci_results.py --source-dir outputs --dataset-id "${CIRCLECI_RESULTS_DATASET_ID}"
+
+      - name: Post comment with helper link
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          github_repository: ${{ github.repository }}
+          pr_number: ${{ github.event.pull_request.number }}
+          pr_sha: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if [ ! -f outputs/failure_summary.json ]; then
+            echo "failure_summary.json missing, skipping comment."
+            exit 0
+          fi
+          failures=$(python -c "import json; print(len(json.load(open('outputs/failure_summary.json'))['failures']))")
+          if [ "$failures" -eq 0 ]; then
+            echo "No failures detected, skipping PR comment."
+            exit 0
+          fi
+          SPACE_SLUG="transformers-community/circleci-test-collection-helper"
+          SPACE_BASE="https://huggingface.co/spaces/${SPACE_SLUG}"
+          repo_enc=$(jq -rn --arg v "${github_repository}" '$v|@uri')
+          pr_enc=$(jq -rn --arg v "${pr_number}" '$v|@uri')
+          sha_enc=$(jq -rn --arg v "${pr_sha}" '$v|@uri')
+          parts=()
+          [ -n "${repo_enc}" ] && parts+=("repo=${repo_enc}")
+          [ -n "${pr_enc}" ] && parts+=("pr=${pr_enc}")
+          [ -n "${sha_enc}" ] && parts+=("sha=${sha_enc}")
+          QUERY=$(IFS="&"; echo "${parts[*]}")
+          if [ -n "$QUERY" ]; then
+            SPACE_URL="${SPACE_BASE}?${QUERY}"
+          else
+            SPACE_URL="${SPACE_BASE}"
+          fi
+          body="View the CircleCI test collection helper for this PR:\n\n${SPACE_URL}"
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "repos/${github_repository}/issues/${pr_number}/comments" \
+            -f body="$body"

scripts/find_circleci_workflow.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2025
+#
+# Utility script to retrieve a CircleCI workflow ID for a given branch and commit SHA.
+#
+# Usage:
+#   python scripts/find_circleci_workflow.py --branch main --sha <commit_sha>
+#
+# Environment:
+#   CIRCLE_TOKEN must be set with a token that has permission to query the CircleCI API.
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from typing import Optional
+
+import requests
+
+
+CIRCLE_API = "https://circleci.com/api/v2"
+PROJECT_SLUG = "gh/huggingface/transformers"
+
+
+def _get_circle_token(token: Optional[str]) -> str:
+    token = token or os.environ.get("CIRCLE_TOKEN") or os.environ.get("CCI_TOKEN") or os.environ.get("CIRCLE_TOKEN")
+    if not token:
+        raise SystemExit("CIRCLE_TOKEN (or CCI_TOKEN / CIRCLE_TOKEN) must be provided.")
+    return token
+
+
+def _request(url: str, token: str, params: Optional[dict] = None) -> dict:
+    response = requests.get(
+        url,
+        params=params,
+        headers={"Circle-Token": token},
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def _find_pipeline_id(branch: str, revision: str, token: str) -> str:
+    url = f"{CIRCLE_API}/project/{PROJECT_SLUG}/pipeline"
+    params = {"branch": branch}
+    pages_checked = 0
+    while True:
+        payload = _request(url, token, params=params)
+        for pipeline in payload.get("items", []):
+            vcs = pipeline.get("vcs") or {}
+            if vcs.get("revision") == revision:
+                return pipeline["id"]
+        next_token = payload.get("next_page_token")
+        if not next_token or pages_checked > 10:
+            break
+        params["page-token"] = next_token
+        pages_checked += 1
+    raise SystemExit(f"Unable to find CircleCI pipeline for branch {branch} and revision {revision}.")
+
+
+def _workflow_has_collection_job(workflow_id: str, token: str) -> bool:
+    jobs = _request(f"{CIRCLE_API}/workflow/{workflow_id}/job", token)
+    return any(job.get("name") == "collection_job" for job in jobs.get("items", []))
+
+
+def _find_workflow_with_collection_job(pipeline_id: str, token: str) -> str:
+    payload = _request(f"{CIRCLE_API}/pipeline/{pipeline_id}/workflow", token)
+    workflows = payload.get("items", [])
+    for workflow in workflows:
+        workflow_id = workflow["id"]
+        if _workflow_has_collection_job(workflow_id, token):
+            return workflow_id
+    if workflows:
+        return workflows[0]["id"]
+    raise SystemExit(f"No workflows found for pipeline {pipeline_id}.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Find CircleCI workflow id for a commit.")
+    parser.add_argument("--branch", required=True, help="Branch name for the CircleCI pipeline.")
+    parser.add_argument("--sha", required=True, help="Commit SHA to match.")
+    parser.add_argument("--token", default=None, help="CircleCI API token.")
+    args = parser.parse_args()
+
+    token = _get_circle_token(args.token)
+    pipeline_id = _find_pipeline_id(args.branch, args.sha, token)
+    workflow_id = _find_workflow_with_collection_job(pipeline_id, token)
+    print(workflow_id)
+
+
+if __name__ == "__main__":
+    main()

src/transformers/models/llama/tokenization_llama.py

@@ -237,6 +237,7 @@ def tokenize(self, text: "TextInput", **kwargs) -> list[str]:
         Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
         first token is special.
         """
+        return super().tokenize(text, **kwargs) # Just to have failures :)
         if self.legacy or len(text) == 0:
             return super().tokenize(text, **kwargs)
 

src/transformers/models/mixtral/modeling_mixtral.py

@@ -282,11 +282,11 @@ def eager_attention_forward(
         causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
         attn_weights = attn_weights + causal_mask
 
-    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
-    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    # attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    # attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
     attn_output = torch.matmul(attn_weights, value_states)
     attn_output = attn_output.transpose(1, 2).contiguous()
-
+    # TODO
     return attn_output, attn_weights