Lintcheck: Update crates and expand CI testset to 200 crates

.github/workflows/lintcheck.yml

@@ -53,18 +53,18 @@ jobs:
       id: cache-json
       uses: actions/cache@v4
       with:
-        path: lintcheck-logs/lintcheck_crates_logs.json
+        path: lintcheck-logs/ci_crates_logs.json
         key: ${{ steps.key.outputs.key }}
 
     - name: Run lintcheck
       if: steps.cache-json.outputs.cache-hit != 'true'
-      run: ./target/debug/lintcheck --format json --warn-all
+      run: ./target/debug/lintcheck --format json --warn-all --crates-toml ./lintcheck/ci_crates.toml
 
     - name: Upload base JSON
       uses: actions/upload-artifact@v4
       with:
         name: base
-        path: lintcheck-logs/lintcheck_crates_logs.json
+        path: lintcheck-logs/ci_crates_logs.json
 
   # Runs lintcheck on the PR and stores the results as an artifact
   head:
@@ -86,13 +86,13 @@ jobs:
       run: cargo build --manifest-path=lintcheck/Cargo.toml
 
     - name: Run lintcheck
-      run: ./target/debug/lintcheck --format json --warn-all
+      run: ./target/debug/lintcheck --format json --warn-all --crates-toml ./lintcheck/ci_crates.toml
 
     - name: Upload head JSON
       uses: actions/upload-artifact@v4
       with:
         name: head
-        path: lintcheck-logs/lintcheck_crates_logs.json
+        path: lintcheck-logs/ci_crates_logs.json
 
   # Retrieves the head and base JSON results and prints the diff to the GH actions step summary
   diff:
@@ -115,4 +115,4 @@ jobs:
       uses: actions/download-artifact@v4
 
     - name: Diff results
-      run: ./target/debug/lintcheck diff {base,head}/lintcheck_crates_logs.json >> $GITHUB_STEP_SUMMARY
+      run: ./target/debug/lintcheck diff {base,head}/ci_crates_logs.json >> $GITHUB_STEP_SUMMARY

lintcheck/ci_crates.toml

@@ -0,0 +1,208 @@
+[crates]
+# Binaries projects
+cargo = {name = "cargo", version = '0.64.0', online_link = 'https://docs.rs/cargo/{version}/src/{file}.html#{line}'}
+ripgrep = {name = "ripgrep", version = '14.1.0'}
+bat = {name = "bat", version = '0.24.0'}
+fend = {name = "fend", version = '1.5.0'}
+mdbook = {name = "mdbook", version = '0.4.40'}
+
+# Bigger crates from ICE issues:
+wasmi = {name = "wasmi", version = '0.35.0'}
+wgpu = {name = "wgpu", version = '0.20.1'}
+bytes = {name = "bytes", version = '1.6.1'}
+skrifa = {name = "skrifa", version = '0.19.3'}
+
+# Random crates which are part of the default test set
+puffin = {name = "puffin", version = '0.19.0'}
+
+# Top ~200 crates from crates.io
+syn = { name = 'syn', version = '2.0.71' }
+bitflags = { name = 'bitflags', version = '2.6.0' }
+hashbrown = { name = 'hashbrown', version = '0.14.5' }
+base64 = { name = 'base64', version = '0.22.1' }
+regex-syntax = { name = 'regex-syntax', version = '0.8.4' }
+proc-macro2 = { name = 'proc-macro2', version = '1.0.86' }
+indexmap = { name = 'indexmap', version = '2.2.6' }
+quote = { name = 'quote', version = '1.0.36' }
+regex-automata = { name = 'regex-automata', version = '0.4.7' }
+libc = { name = 'libc', version = '0.2.155' }
+serde = { name = 'serde', version = '1.0.204' }
+itertools = { name = 'itertools', version = '0.13.0' }
+heck = { name = 'heck', version = '0.5.0' }
+memchr = { name = 'memchr', version = '2.7.4' }
+serde_derive = { name = 'serde_derive', version = '1.0.204' }
+unicode-ident = { name = 'unicode-ident', version = '1.0.12' }
+autocfg = { name = 'autocfg', version = '1.3.0' }
+cfg-if = { name = 'cfg-if', version = '1.0.0' }
+aho-corasick = { name = 'aho-corasick', version = '1.1.3' }
+getrandom = { name = 'getrandom', version = '0.2.15' }
+rand_core = { name = 'rand_core', version = '0.6.4' }
+serde_json = { name = 'serde_json', version = '1.0.120' }
+itoa = { name = 'itoa', version = '1.0.11' }
+rand = { name = 'rand', version = '0.8.5' }
+ryu = { name = 'ryu', version = '1.0.18' }
+once_cell = { name = 'once_cell', version = '1.19.0' }
+rustix = { name = 'rustix', version = '0.38.34' }
+regex = { name = 'regex', version = '1.10.5' }
+log = { name = 'log', version = '0.4.22' }
+parking_lot_core = { name = 'parking_lot_core', version = '0.9.10' }
+cc = { name = 'cc', version = '1.1.5' }
+strsim = { name = 'strsim', version = '0.11.1' }
+clap = { name = 'clap', version = '4.5.9' }
+parking_lot = { name = 'parking_lot', version = '0.12.3' }
+smallvec = { name = 'smallvec', version = '2.0.0-alpha.6' }
+thiserror-impl = { name = 'thiserror-impl', version = '1.0.63' }
+thiserror = { name = 'thiserror', version = '1.0.63' }
+linux-raw-sys = { name = 'linux-raw-sys', version = '0.6.4' }
+socket2 = { name = 'socket2', version = '0.5.7' }
+idna = { name = 'idna', version = '1.0.2' }
+fastrand = { name = 'fastrand', version = '2.1.0' }
+either = { name = 'either', version = '1.13.0' }
+num-traits = { name = 'num-traits', version = '0.2.19' }
+rand_chacha = { name = 'rand_chacha', version = '0.3.1' }
+lazy_static = { name = 'lazy_static', version = '1.5.0' }
+semver = { name = 'semver', version = '1.0.23' }
+lock_api = { name = 'lock_api', version = '0.4.12' }
+scopeguard = { name = 'scopeguard', version = '1.2.0' }
+ahash = { name = 'ahash', version = '0.8.11' }
+anyhow = { name = 'anyhow', version = '1.0.86' }
+rustls = { name = 'rustls', version = '0.23.11' }
+http = { name = 'http', version = '1.1.0' }
+toml_edit = { name = 'toml_edit', version = '0.22.16' }
+pin-project-lite = { name = 'pin-project-lite', version = '0.2.14' }
+spin = { name = 'spin', version = '0.9.8' }
+miniz_oxide = { name = 'miniz_oxide', version = '0.7.4' }
+memoffset = { name = 'memoffset', version = '0.9.1' }
+digest = { name = 'digest', version = '0.11.0-pre.8' }
+version_check = { name = 'version_check', version = '0.9.4' }
+clap_lex = { name = 'clap_lex', version = '0.7.1' }
+crossbeam-utils = { name = 'crossbeam-utils', version = '0.8.20' }
+toml = { name = 'toml', version = '0.8.15' }
+block-buffer = { name = 'block-buffer', version = '0.10.4' }
+time = { name = 'time', version = '0.3.36' }
+hyper = { name = 'hyper', version = '1.4.1' }
+url = { name = 'url', version = '2.5.2' }
+percent-encoding = { name = 'percent-encoding', version = '2.3.1' }
+tokio = { name = 'tokio', version = '1.38.1' }
+errno = { name = 'errno', version = '0.3.9' }
+uuid = { name = 'uuid', version = '1.10.0' }
+unicode-normalization = { name = 'unicode-normalization', version = '0.1.23' }
+ppv-lite86 = { name = 'ppv-lite86', version = '0.2.17' }
+futures-core = { name = 'futures-core', version = '0.3.30' }
+http-body = { name = 'http-body', version = '1.0.1' }
+tinyvec = { name = 'tinyvec', version = '1.8.0' }
+futures-util = { name = 'futures-util', version = '0.3.30' }
+futures-task = { name = 'futures-task', version = '0.3.30' }
+sha2 = { name = 'sha2', version = '0.11.0-pre.3' }
+ring = { name = 'ring', version = '0.17.8' }
+slab = { name = 'slab', version = '0.4.9' }
+chrono = { name = 'chrono', version = '0.4.38' }
+futures-sink = { name = 'futures-sink', version = '0.3.30' }
+futures-channel = { name = 'futures-channel', version = '0.3.30' }
+num_cpus = { name = 'num_cpus', version = '1.16.0' }
+untrusted = { name = 'untrusted', version = '0.9.0' }
+tinyvec_macros = { name = 'tinyvec_macros', version = '0.1.1' }
+mio = { name = 'mio', version = '1.0.0' }
+byteorder = { name = 'byteorder', version = '1.5.0' }
+form_urlencoded = { name = 'form_urlencoded', version = '1.2.1' }
+unicode-bidi = { name = 'unicode-bidi', version = '0.3.15' }
+futures-io = { name = 'futures-io', version = '0.3.30' }
+tokio-util = { name = 'tokio-util', version = '0.7.11' }
+rustls-pemfile = { name = 'rustls-pemfile', version = '2.1.2' }
+generic-array = { name = 'generic-array', version = '1.1.0' }
+tracing = { name = 'tracing', version = '0.1.40' }
+equivalent = { name = 'equivalent', version = '1.0.1' }
+tracing-core = { name = 'tracing-core', version = '0.1.32' }
+pin-utils = { name = 'pin-utils', version = '0.1.0' }
+tempfile = { name = 'tempfile', version = '3.10.1' }
+h2 = { name = 'h2', version = '0.4.5' }
+futures = { name = 'futures', version = '0.3.30' }
+typenum = { name = 'typenum', version = '1.17.0' }
+winnow = { name = 'winnow', version = '0.6.13' }
+cpufeatures = { name = 'cpufeatures', version = '0.2.12' }
+nix = { name = 'nix', version = '0.29.0' }
+fnv = { name = 'fnv', version = '1.0.7' }
+tokio-rustls = { name = 'tokio-rustls', version = '0.26.0' }
+iana-time-zone = { name = 'iana-time-zone', version = '0.1.60' }
+rustls-webpki = { name = 'rustls-webpki', version = '0.102.5' }
+crc32fast = { name = 'crc32fast', version = '1.4.2' }
+adler = { name = 'adler', version = '1.0.2' }
+pkg-config = { name = 'pkg-config', version = '0.3.30' }
+redox_syscall = { name = 'redox_syscall', version = '0.5.3' }
+nom = { name = 'nom', version = '8.0.0-alpha2' }
+rustc_version = { name = 'rustc_version', version = '0.4.0' }
+futures-macro = { name = 'futures-macro', version = '0.3.30' }
+clap_derive = { name = 'clap_derive', version = '4.5.8' }
+futures-executor = { name = 'futures-executor', version = '0.3.30' }
+event-listener = { name = 'event-listener', version = '5.3.1' }
+num-integer = { name = 'num-integer', version = '0.1.46' }
+time-macros = { name = 'time-macros', version = '0.2.18' }
+flate2 = { name = 'flate2', version = '1.0.30' }
+tokio-macros = { name = 'tokio-macros', version = '2.3.0' }
+strum_macros = { name = 'strum_macros', version = '0.26.4' }
+tracing-attributes = { name = 'tracing-attributes', version = '0.1.27' }
+async-trait = { name = 'async-trait', version = '0.1.81' }
+crypto-common = { name = 'crypto-common', version = '0.1.6' }
+unicode-width = { name = 'unicode-width', version = '0.1.13' }
+anstyle = { name = 'anstyle', version = '1.0.7' }
+object = { name = 'object', version = '0.36.1' }
+gimli = { name = 'gimli', version = '0.31.0' }
+crossbeam-epoch = { name = 'crossbeam-epoch', version = '0.9.18' }
+thread_local = { name = 'thread_local', version = '1.1.8' }
+strum = { name = 'strum', version = '0.26.3' }
+darling_core = { name = 'darling_core', version = '0.20.10' }
+darling_macro = { name = 'darling_macro', version = '0.20.10' }
+minimal-lexical = { name = 'minimal-lexical', version = '0.2.1' }
+clap_builder = { name = 'clap_builder', version = '4.5.9' }
+time-core = { name = 'time-core', version = '0.1.2' }
+httparse = { name = 'httparse', version = '1.9.4' }
+signal-hook-registry = { name = 'signal-hook-registry', version = '1.4.2' }
+hex = { name = 'hex', version = '0.4.3' }
+crossbeam-deque = { name = 'crossbeam-deque', version = '0.8.5' }
+zerocopy = { name = 'zerocopy', version = '0.7.35' }
+rustversion = { name = 'rustversion', version = '1.0.17' }
+env_logger = { name = 'env_logger', version = '0.11.3' }
+webpki-roots = { name = 'webpki-roots', version = '0.26.3' }
+rustc-demangle = { name = 'rustc-demangle', version = '0.1.24' }
+mime = { name = 'mime', version = '0.3.17' }
+termcolor = { name = 'termcolor', version = '1.4.1' }
+subtle = { name = 'subtle', version = '2.6.1' }
+walkdir = { name = 'walkdir', version = '2.5.0' }
+hermit-abi = { name = 'hermit-abi', version = '0.4.0' }
+pin-project = { name = 'pin-project', version = '1.1.5' }
+pin-project-internal = { name = 'pin-project-internal', version = '1.1.5' }
+try-lock = { name = 'try-lock', version = '0.2.5' }
+tracing-log = { name = 'tracing-log', version = '0.2.0' }
+httpdate = { name = 'httpdate', version = '1.0.3' }
+anstream = { name = 'anstream', version = '0.6.14' }
+crossbeam-channel = { name = 'crossbeam-channel', version = '0.5.13' }
+reqwest = { name = 'reqwest', version = '0.12.5' }
+want = { name = 'want', version = '0.3.1' }
+paste = { name = 'paste', version = '1.0.15' }
+anstyle-parse = { name = 'anstyle-parse', version = '0.2.4' }
+toml_datetime = { name = 'toml_datetime', version = '0.6.6' }
+anstyle-query = { name = 'anstyle-query', version = '1.1.0' }
+addr2line = { name = 'addr2line', version = '0.24.0' }
+glob = { name = 'glob', version = '0.3.1' }
+num-bigint = { name = 'num-bigint', version = '0.4.6' }
+backtrace = { name = 'backtrace', version = '0.3.73' }
+wasi = { name = 'wasi', version = '0.13.1+wasi-0.2.0' }
+tower-service = { name = 'tower-service', version = '0.3.2' }
+sync_wrapper = { name = 'sync_wrapper', version = '1.0.1' }
+libloading = { name = 'libloading', version = '0.8.4' }
+rayon = { name = 'rayon', version = '1.10.0' }
+colorchoice = { name = 'colorchoice', version = '1.0.1' }
+encoding_rs = { name = 'encoding_rs', version = '0.8.34' }
+deranged = { name = 'deranged', version = '0.3.11' }
+zeroize = { name = 'zeroize', version = '1.8.1' }
+utf8parse = { name = 'utf8parse', version = '0.2.2' }
+tracing-subscriber = { name = 'tracing-subscriber', version = '0.3.18' }
+hyper-rustls = { name = 'hyper-rustls', version = '0.27.2' }
+hmac = { name = 'hmac', version = '0.13.0-pre.3' }
+rayon-core = { name = 'rayon-core', version = '1.12.1' }
+same-file = { name = 'same-file', version = '1.0.6' }
+prost = { name = 'prost', version = '0.13.1' }
+sharded-slab = { name = 'sharded-slab', version = '0.1.7' }
+textwrap = { name = 'textwrap', version = '0.16.1' }
+bumpalo = {name = "bumpalo", version = '3.16.0'}
+arrayvec = { name = 'arrayvec', version = '0.7.4' }

lintcheck/lintcheck_crates.toml

@@ -1,38 +1,44 @@
+# If you want to check a local project it's usually easier to use:
+# ```
+# cargo dev lint <path>
+# ```
+#
+# For testing you can also add sources to git and local repos like this:
+# ```
+# crate = {name = "crate", git_url = "https://github.com/name/repo.git", git_hash = "coo1cafe"}
+# crate = {name = "crate", path = "/path/to/project"}
+# ```
+
 [crates]
-# some of these are from cargotest
-cargo = {name = "cargo", version = '0.64.0', online_link = 'https://docs.rs/cargo/{version}/src/{file}.html#{line}'}
-iron = {name = "iron", version = '0.6.1'}
-ripgrep = {name = "ripgrep", version = '12.1.1'}
-xsv = {name = "xsv", version = '0.13.0'}
-# commented out because of 173K clippy::match_same_arms msgs in language_type.rs
-#tokei = { name = "tokei", version = '12.0.4'}
-rayon = {name = "rayon", version = '1.5.0'}
-serde = {name = "serde", version = '1.0.118'}
-# top 10 crates.io dls
-bitflags = {name = "bitflags", version = '1.2.1'}
-# crash = {name = "clippy_crash", path = "/tmp/clippy_crash"}
-libc = {name = "libc", version = '0.2.81'}
-log = {name = "log", version = '0.4.11'}
-proc-macro2 = {name = "proc-macro2", version = '1.0.24'}
-quote = {name = "quote", version = '1.0.7'}
-rand = {name = "rand", version = '0.7.3'}
-rand_core = {name = "rand_core", version = '0.6.0'}
-regex = {name = "regex", version = '1.3.2'}
-syn = {name = "syn", version = '1.0.54'}
-unicode-xid = {name = "unicode-xid", version = '0.2.1'}
-# some more of dtolnays crates
-anyhow = {name = "anyhow", version = '1.0.38'}
-async-trait = {name = "async-trait", version = '0.1.42'}
-cxx = {name = "cxx", version = '1.0.32'}
-ryu = {name = "ryu", version = '1.0.5'}
-serde_yaml = {name = "serde_yaml", version = '0.8.17'}
-thiserror = {name = "thiserror", version = '1.0.24'}
-# some embark crates, there are other interesting crates but
-# unfortunately adding them increases lintcheck runtime drastically
-cfg-expr = {name = "cfg-expr", version = '0.7.1'}
-puffin = {name = "puffin", git_url = "https://github.com/EmbarkStudios/puffin", git_hash = "02dd4a3"}
-rpmalloc = {name = "rpmalloc", version = '0.2.0'}
-tame-oidc = {name = "tame-oidc", version = '0.1.0'}
+
+# Some binaries
+cargo = {name = "cargo", version = '0.80.0', online_link = 'https://docs.rs/cargo/{version}/src/{file}.html#{line}'}
+ripgrep = {name = "ripgrep", version = '14.1.0'}
+mdbook = {name = "mdbook", version = '0.4.40'}
+
+# Common libraries
+rayon = {name = "rayon", version = '1.10.0'}
+serde = {name = "serde", version = '1.0.204'}
+bitflags = {name = "bitflags", version = '2.6.0'}
+log = {name = "log", version = '0.4.22'}
+quote = {name = "quote", version = '1.0.36'}
+proc-macro2 = {name = "proc-macro2", version = '1.0.86'}
+rand = {name = "rand", version = '0.8.5'}
+rand_core = {name = "rand_core", version = '0.6.4'}
+regex = {name = "regex", version = '1.10.5'}
+syn = {name = "syn", version = '2.0.71'}
+anyhow = {name = "anyhow", version = '1.0.86'}
+async-trait = { name = 'async-trait', version = '0.1.81' }
+cxx = {name = "cxx", version = '1.0.124'}
+ryu = {name = "ryu", version = '1.0.18'}
+thiserror = {name = "thiserror", version = '1.0.63'}
+serde_yaml = {name = "serde_yaml", version = '0.9.33'}
+puffin = {name = "puffin", version = '0.19.0'}
+bumpalo = {name = "bumpalo", version = '3.16.0'}
+wasmi = {name = "wasmi", version = '0.35.0'}
+base64 = { name = 'base64', version = '0.22.1' }
+once_cell = { name = 'once_cell', version = '1.19.0' }
+tokio = { name = 'tokio', version = '1.38.1' }
 
 [recursive]
 ignore = [

lintcheck/src/input.rs

@@ -159,11 +159,25 @@ pub fn read_crates(toml_path: &Path) -> (Vec<CrateWithSource>, RecursiveOptions)
 }
 
 impl CrateWithSource {
+    pub fn download_and_prepare(&self) -> Crate {
+        let krate = self.download_and_extract();
+
+        // Downloaded crates might contain a `rust-toolchain` file. This file
+        // seems to be accessed when `build.rs` files are present. This access
+        // results in build errors since lintcheck and clippy will most certainly
+        // use a different toolchain.
+        // Lintcheck simply removes these files and assumes that our toolchain
+        // is more up to date.
+        let _ = fs::remove_file(krate.path.join("rust-toolchain"));
+        let _ = fs::remove_file(krate.path.join("rust-toolchain.toml"));
+
+        krate
+    }
     /// Makes the sources available on the disk for clippy to check.
     /// Clones a git repo and checks out the specified commit or downloads a crate from crates.io or
     /// copies a local folder
     #[expect(clippy::too_many_lines)]
-    pub fn download_and_extract(&self) -> Crate {
+    fn download_and_extract(&self) -> Crate {
         #[allow(clippy::result_large_err)]
         fn get(path: &str) -> Result<ureq::Response, ureq::Error> {
             const MAX_RETRIES: u8 = 4;