compiletest/
lib.rs

1#![crate_name = "compiletest"]
2#![warn(unreachable_pub)]
3
4#[cfg(test)]
5mod tests;
6
7// Public modules needed by the compiletest binary or by `rustdoc-gui-test`.
8pub mod cli;
9pub mod rustdoc_gui_test;
10
11mod common;
12mod debuggers;
13mod diagnostics;
14mod directives;
15mod edition;
16mod errors;
17mod executor;
18mod json;
19mod output_capture;
20mod panic_hook;
21mod raise_fd_limit;
22mod read2;
23mod runtest;
24mod util;
25
26use core::panic;
27use std::collections::HashSet;
28use std::fmt::Write;
29use std::io::{self, ErrorKind};
30use std::sync::{Arc, OnceLock};
31use std::time::SystemTime;
32use std::{env, fs, vec};
33
34use build_helper::git::{get_git_modified_files, get_git_untracked_files};
35use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
36use clap::Parser;
37use rayon::iter::{ParallelBridge, ParallelIterator};
38use tracing::debug;
39use walkdir::WalkDir;
40
41use self::directives::{EarlyProps, make_test_description};
42use crate::common::{
43    CodegenBackend, CompareMode, Config, Debugger, ForcePassMode, TestMode, TestPaths, TestSuite,
44    UI_EXTENSIONS, expected_output_path, output_base_dir, output_relative_path,
45};
46use crate::directives::{AuxProps, DirectivesCache, FileDirectives};
47use crate::edition::Edition;
48use crate::executor::CollectedTest;
49
50/// Compiletest command-line arguments.
51#[derive(clap::Parser)]
52struct Args {
53    // Required options
54    /// Path to host shared libraries.
55    #[arg(long)]
56    compile_lib_path: Utf8PathBuf,
57    /// Path to target shared libraries.
58    #[arg(long)]
59    run_lib_path: Utf8PathBuf,
60    /// Path to rustc to use for compiling.
61    #[arg(long)]
62    rustc_path: Utf8PathBuf,
63    /// Path to python to use for doc tests.
64    #[arg(long)]
65    python: String,
66    /// Directory containing sources.
67    #[arg(long)]
68    src_root: Utf8PathBuf,
69    /// Directory containing test suite sources.
70    #[arg(long)]
71    src_test_suite_root: Utf8PathBuf,
72    /// Path to root build directory.
73    #[arg(long)]
74    build_root: Utf8PathBuf,
75    /// Path to test suite specific build directory.
76    #[arg(long)]
77    build_test_suite_root: Utf8PathBuf,
78    /// Directory containing the compiler sysroot.
79    #[arg(long)]
80    sysroot_base: Utf8PathBuf,
81    /// Stage number under test.
82    #[arg(long)]
83    stage: u32,
84    /// The target-stage identifier.
85    #[arg(long)]
86    stage_id: String,
87    /// Which sort of compile tests to run.
88    #[arg(long)]
89    mode: TestMode,
90    /// Which suite of compile tests to run.
91    #[arg(long)]
92    suite: TestSuite,
93    /// Path to a C compiler.
94    #[arg(long)]
95    cc: String,
96    /// Path to a C++ compiler.
97    #[arg(long)]
98    cxx: String,
99    /// Flags for the C compiler.
100    #[arg(long, allow_hyphen_values = true)]
101    cflags: String,
102    /// Flags for the CXX compiler.
103    #[arg(long, allow_hyphen_values = true)]
104    cxxflags: String,
105    /// List of LLVM components built in.
106    #[arg(long)]
107    llvm_components: String,
108    /// Current Rust channel.
109    #[arg(long)]
110    channel: String,
111    /// Name of the git branch for nightly.
112    #[arg(long)]
113    nightly_branch: String,
114    /// Email address used for finding merge commits.
115    #[arg(long)]
116    git_merge_commit_email: String,
117    /// Path to minicore aux library.
118    #[arg(long)]
119    minicore_path: Utf8PathBuf,
120    /// Number of parallel jobs bootstrap was configured with.
121    #[arg(long)]
122    jobs: u32,
123    /// The host to build for.
124    #[arg(long)]
125    host: String,
126    /// The target to build for.
127    #[arg(long)]
128    target: String,
129
130    // Optional options
131    /// Path to cargo to use for compiling.
132    #[arg(long)]
133    cargo_path: Option<Utf8PathBuf>,
134    /// Path to rustc to use for compiling run-make recipes.
135    #[arg(long)]
136    stage0_rustc_path: Option<Utf8PathBuf>,
137    /// Path to rustc to use for querying target information.
138    #[arg(long)]
139    query_rustc_path: Option<Utf8PathBuf>,
140    /// Path to rustdoc to use for compiling.
141    #[arg(long)]
142    rustdoc_path: Option<Utf8PathBuf>,
143    /// Path to coverage-dump to use in tests.
144    #[arg(long)]
145    coverage_dump_path: Option<Utf8PathBuf>,
146    /// Path to jsondocck to use for doc tests.
147    #[arg(long)]
148    jsondocck_path: Option<Utf8PathBuf>,
149    /// Path to jsondoclint to use for doc tests.
150    #[arg(long)]
151    jsondoclint_path: Option<Utf8PathBuf>,
152    /// Path to Clang executable.
153    #[arg(long)]
154    run_clang_based_tests_with: Option<Utf8PathBuf>,
155    /// Path to LLVM's FileCheck binary.
156    #[arg(long)]
157    llvm_filecheck: Option<Utf8PathBuf>,
158    /// Path to LLVM's bin directory.
159    #[arg(long)]
160    llvm_bin_dir: Option<Utf8PathBuf>,
161    /// The name of nodejs.
162    #[arg(long)]
163    nodejs: Option<Utf8PathBuf>,
164    /// The name of npm.
165    #[arg(long)]
166    npm: Option<Utf8PathBuf>,
167    /// Path to the remote test client.
168    #[arg(long)]
169    remote_test_client: Option<Utf8PathBuf>,
170    /// Path to CDB to use for CDB debuginfo tests.
171    #[arg(long)]
172    cdb: Option<Utf8PathBuf>,
173    /// Path to GDB to use for GDB debuginfo tests.
174    #[arg(long)]
175    gdb: Option<Utf8PathBuf>,
176    /// Path to LLDB to use for LLDB debuginfo tests.
177    #[arg(long)]
178    lldb: Option<Utf8PathBuf>,
179    /// The version of LLDB used.
180    #[arg(long)]
181    lldb_version: Option<String>,
182    /// The version of LLVM used.
183    #[arg(long)]
184    llvm_version: Option<String>,
185    /// Android NDK standalone path.
186    #[arg(long)]
187    android_cross_path: Option<Utf8PathBuf>,
188    /// Path to the android debugger.
189    #[arg(long)]
190    adb_path: Option<Utf8PathBuf>,
191    /// Path to tests for the android debugger.
192    #[arg(long)]
193    adb_test_dir: Option<Utf8PathBuf>,
194    /// Path to an archiver.
195    #[arg(long, default_value = "ar")]
196    ar: String,
197    /// Path to a linker for the target.
198    #[arg(long)]
199    target_linker: Option<String>,
200    /// Path to a linker for the host.
201    #[arg(long)]
202    host_linker: Option<String>,
203    /// Force {check,build,run}-pass tests to this mode.
204    #[arg(long)]
205    pass: Option<ForcePassMode>,
206    /// Whether to execute run-* tests.
207    #[arg(long)]
208    run: Option<String>,
209    /// Supervisor program to run tests under (eg. emulator, valgrind).
210    #[arg(long)]
211    runner: Option<String>,
212    /// Mode describing what file the actual ui output will be compared to.
213    #[arg(long)]
214    compare_mode: Option<CompareMode>,
215    /// Default Rust edition.
216    #[arg(long)]
217    edition: Option<Edition>,
218    /// Only test a specific debugger in debuginfo tests.
219    #[arg(long)]
220    debugger: Option<String>,
221    /// The codegen backend currently used.
222    #[arg(long)]
223    default_codegen_backend: Option<CodegenBackend>,
224    /// The codegen backend to use instead of the default one.
225    #[arg(long)]
226    override_codegen_backend: Option<String>,
227    /// Custom diff tool to use for displaying compiletest tests.
228    #[arg(long)]
229    compiletest_diff_tool: Option<String>,
230    /// Number of parallel threads to use for the frontend when building test artifacts
231    #[arg(long)]
232    parallel_frontend_threads: Option<u32>,
233    /// Number of times to execute each test.
234    #[arg(long)]
235    iteration_count: Option<u32>,
236
237    // Flags
238    /// Overwrite stderr/stdout files instead of complaining about a mismatch.
239    #[arg(long)]
240    bless: bool,
241    /// Stop as soon as possible after any test fails.
242    #[arg(long)]
243    fail_fast: bool,
244    /// Run tests marked as ignored.
245    #[arg(long)]
246    ignored: bool,
247    /// Run tests that require enzyme.
248    #[arg(long)]
249    has_enzyme: bool,
250    /// Run tests that require offload.
251    #[arg(long)]
252    has_offload: bool,
253    /// Whether rustc was built with debug assertions.
254    #[arg(long)]
255    with_rustc_debug_assertions: bool,
256    /// Whether std was built with debug assertions.
257    #[arg(long)]
258    with_std_debug_assertions: bool,
259    /// Whether std was built with remapping.
260    #[arg(long)]
261    with_std_remap_debuginfo: bool,
262    /// Filters match exactly.
263    #[arg(long)]
264    exact: bool,
265    /// Set this when rustc/stdlib were compiled with randomized layouts.
266    #[arg(long)]
267    rust_randomized_layout: bool,
268    /// Run tests with optimizations enabled.
269    #[arg(long)]
270    optimize_tests: bool,
271    /// Run tests verbosely, showing all output.
272    #[arg(long)]
273    verbose: bool,
274    /// Show verbose subprocess output for successful run-make tests.
275    #[arg(long)]
276    verbose_run_make_subprocess_output: bool,
277    /// Is LLVM the system LLVM.
278    #[arg(long)]
279    system_llvm: bool,
280    /// Rerun tests even if the inputs are unchanged.
281    #[arg(long)]
282    force_rerun: bool,
283    /// Only run tests that result been modified.
284    #[arg(long)]
285    only_modified: bool,
286    // Backcompat option
287    #[arg(long, hide = true)]
288    nocapture: bool,
289    /// Don't capture stdout/stderr of tests.
290    #[arg(long)]
291    no_capture: bool,
292    /// Is the profiler runtime enabled for this target.
293    #[arg(long)]
294    profiler_runtime: bool,
295    /// Run tests which rely on commit version being compiled into the binaries.
296    #[arg(long)]
297    git_hash: bool,
298    /// Enable this to generate a Rustfix coverage file.
299    #[arg(long)]
300    rustfix_coverage: bool,
301    /// Ignore `//@ ignore-backends` directives.
302    #[arg(long)]
303    bypass_ignore_backends: bool,
304
305    // These values can be entered multiple times, for example:
306    // --skip foo --skip bar
307    /// Skip tests matching SUBSTRING.
308    #[arg(long)]
309    skip: Vec<String>,
310    /// Flags to pass to rustc for host.
311    #[arg(long, allow_hyphen_values = true)]
312    host_rustcflags: Vec<String>,
313    /// Flags to pass to rustc for target.
314    #[arg(long, allow_hyphen_values = true)]
315    target_rustcflags: Vec<String>,
316
317    // Positional arguments
318    /// Test name filters.
319    /// All leftover arguments will be stored in this list.
320    filters: Vec<String>,
321}
322
323fn parse_config(args: Vec<String>) -> Config {
324    let args = Args::parse_from(args);
325
326    fn make_absolute(path: Utf8PathBuf) -> Utf8PathBuf {
327        if path.is_relative() {
328            Utf8PathBuf::try_from(env::current_dir().unwrap()).unwrap().join(path)
329        } else {
330            path
331        }
332    }
333
334    if args.nocapture {
335        panic!("`--nocapture` is deprecated; please use `--no-capture`");
336    }
337
338    let adb_device_status = args.target.contains("android") && args.adb_test_dir.is_some();
339
340    // FIXME: `cdb_version` is *derived* from cdb, but it's *not* technically a config!
341    let cdb_version = args.cdb.as_deref().and_then(debuggers::query_cdb_version);
342    // FIXME: `gdb_version` is *derived* from gdb, but it's *not* technically a config!
343    let gdb_version = args.gdb.as_deref().and_then(debuggers::query_gdb_version);
344    // FIXME: `lldb_version` is *derived* from lldb, but it's *not* technically a config!
345    let lldb_version = args.lldb_version.as_deref().and_then(debuggers::extract_lldb_version);
346    // FIXME: this is very questionable, we really should be obtaining LLVM version info from
347    // `bootstrap`, and not trying to be figuring out that in `compiletest` by running the
348    // `FileCheck` binary.
349    let llvm_version =
350        args.llvm_version.as_deref().map(directives::extract_llvm_version).or_else(|| {
351            directives::extract_llvm_version_from_binary(args.llvm_filecheck.as_ref()?.as_str())
352        });
353
354    let default_codegen_backend = args.default_codegen_backend.unwrap_or(CodegenBackend::Llvm);
355
356    let mode = args.mode;
357    let filters = if mode == TestMode::RunMake {
358        args.filters
359            .iter()
360            .map(|f| {
361                // Here `f` is relative to `./tests/run-make`. So if you run
362                //
363                //   ./x test tests/run-make/crate-loading
364                //
365                //  then `f` is "crate-loading".
366                let path = Utf8Path::new(f);
367                let mut iter = path.iter().skip(1);
368
369                if iter.next().is_some_and(|s| s == "rmake.rs") && iter.next().is_none() {
370                    // Strip the "rmake.rs" suffix. For example, if `f` is
371                    // "crate-loading/rmake.rs" then this gives us "crate-loading".
372                    path.parent().unwrap().to_string()
373                } else {
374                    f.to_string()
375                }
376            })
377            .collect::<Vec<_>>()
378    } else {
379        // Note that the filters are relative to the root dir of the different test
380        // suites. For example, with:
381        //
382        //   ./x test tests/ui/lint/unused
383        //
384        // the filter is "lint/unused".
385        args.filters.clone()
386    };
387
388    let compare_mode = args.compare_mode;
389
390    let src_root = args.src_root;
391    let src_test_suite_root = args.src_test_suite_root;
392    assert!(
393        src_test_suite_root.starts_with(&src_root),
394        "`src-root` must be a parent of `src-test-suite-root`: `src-root`=`{}`, `src-test-suite-root` = `{}`",
395        src_root,
396        src_test_suite_root
397    );
398
399    let build_root = args.build_root;
400    let build_test_suite_root = args.build_test_suite_root;
401    assert!(build_test_suite_root.starts_with(&build_root));
402
403    let parallel_frontend_threads =
404        args.parallel_frontend_threads.unwrap_or(Config::DEFAULT_PARALLEL_FRONTEND_THREADS);
405    let iteration_count = args.iteration_count.unwrap_or(Config::DEFAULT_ITERATION_COUNT);
406    assert!(iteration_count > 0, "`--iteration-count` must be a positive integer");
407
408    Config {
409        bless: args.bless,
410        fail_fast: args.fail_fast || env::var_os("RUSTC_TEST_FAIL_FAST").is_some(),
411
412        host_compile_lib_path: make_absolute(args.compile_lib_path),
413        target_run_lib_path: make_absolute(args.run_lib_path),
414        rustc_path: args.rustc_path,
415        cargo_path: args.cargo_path,
416        stage0_rustc_path: args.stage0_rustc_path,
417        query_rustc_path: args.query_rustc_path,
418        rustdoc_path: args.rustdoc_path,
419        coverage_dump_path: args.coverage_dump_path,
420        python: args.python,
421        jsondocck_path: args.jsondocck_path,
422        jsondoclint_path: args.jsondoclint_path,
423        run_clang_based_tests_with: args.run_clang_based_tests_with,
424        llvm_filecheck: args.llvm_filecheck,
425        llvm_bin_dir: args.llvm_bin_dir,
426
427        src_root,
428        src_test_suite_root,
429
430        build_root,
431        build_test_suite_root,
432
433        sysroot_base: args.sysroot_base,
434
435        stage: args.stage,
436        stage_id: args.stage_id,
437
438        mode,
439        suite: args.suite,
440        debugger: args.debugger.map(|debugger| {
441            debugger
442                .parse::<Debugger>()
443                .unwrap_or_else(|_| panic!("unknown `--debugger` option `{debugger}` given"))
444        }),
445        run_ignored: args.ignored,
446        with_rustc_debug_assertions: args.with_rustc_debug_assertions,
447        with_std_debug_assertions: args.with_std_debug_assertions,
448        with_std_remap_debuginfo: args.with_std_remap_debuginfo,
449        filters,
450        skip: args.skip,
451        filter_exact: args.exact,
452        force_pass_mode: args.pass,
453        // FIXME: this run scheme is... confusing.
454        run: args.run.and_then(|mode| match mode.as_str() {
455            "auto" => None,
456            "always" => Some(true),
457            "never" => Some(false),
458            _ => panic!("unknown `--run` option `{}` given", mode),
459        }),
460        runner: args.runner,
461        host_rustcflags: args.host_rustcflags,
462        target_rustcflags: args.target_rustcflags,
463        optimize_tests: args.optimize_tests,
464        rust_randomized_layout: args.rust_randomized_layout,
465        target: args.target,
466        host: args.host,
467        cdb: args.cdb,
468        cdb_version,
469        gdb: args.gdb,
470        gdb_version,
471        lldb: args.lldb,
472        lldb_version,
473        llvm_version,
474        system_llvm: args.system_llvm,
475        android_cross_path: args.android_cross_path,
476        adb_path: args.adb_path,
477        adb_test_dir: args.adb_test_dir,
478        adb_device_status,
479        verbose: args.verbose,
480        verbose_run_make_subprocess_output: args.verbose_run_make_subprocess_output,
481        only_modified: args.only_modified,
482        remote_test_client: args.remote_test_client,
483        compare_mode,
484        rustfix_coverage: args.rustfix_coverage,
485        has_enzyme: args.has_enzyme,
486        has_offload: args.has_offload,
487        channel: args.channel,
488        git_hash: args.git_hash,
489        edition: args.edition,
490
491        cc: args.cc,
492        cxx: args.cxx,
493        cflags: args.cflags,
494        cxxflags: args.cxxflags,
495        ar: args.ar,
496        target_linker: args.target_linker,
497        host_linker: args.host_linker,
498        llvm_components: args.llvm_components,
499        nodejs: args.nodejs,
500
501        force_rerun: args.force_rerun,
502
503        target_cfgs: OnceLock::new(),
504        builtin_cfg_names: OnceLock::new(),
505        supported_crate_types: OnceLock::new(),
506
507        capture: !args.no_capture,
508
509        nightly_branch: args.nightly_branch,
510        git_merge_commit_email: args.git_merge_commit_email,
511
512        profiler_runtime: args.profiler_runtime,
513
514        diff_command: args.compiletest_diff_tool,
515
516        minicore_path: args.minicore_path,
517
518        default_codegen_backend,
519        override_codegen_backend: args.override_codegen_backend,
520        bypass_ignore_backends: args.bypass_ignore_backends,
521
522        jobs: args.jobs,
523
524        parallel_frontend_threads,
525        iteration_count,
526    }
527}
528
529/// Called by `main` after the config has been parsed.
530fn run_tests(config: Arc<Config>) {
531    debug!(?config, "run_tests");
532
533    panic_hook::install_panic_hook();
534
535    // If we want to collect rustfix coverage information,
536    // we first make sure that the coverage file does not exist.
537    // It will be created later on.
538    if config.rustfix_coverage {
539        let mut coverage_file_path = config.build_test_suite_root.clone();
540        coverage_file_path.push("rustfix_missing_coverage.txt");
541        if coverage_file_path.exists() {
542            if let Err(e) = fs::remove_file(&coverage_file_path) {
543                panic!("Could not delete {} due to {}", coverage_file_path, e)
544            }
545        }
546    }
547
548    // sadly osx needs some file descriptor limits raised for running tests in
549    // parallel (especially when we have lots and lots of child processes).
550    // For context, see #8904
551    unsafe {
552        raise_fd_limit::raise_fd_limit();
553    }
554    // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
555    // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
556    //
557    // SAFETY: at this point we're still single-threaded.
558    unsafe { env::set_var("__COMPAT_LAYER", "RunAsInvoker") };
559
560    let mut configs = Vec::new();
561    if let TestMode::DebugInfo = config.mode {
562        // Debugging emscripten code doesn't make sense today
563        if !config.target.contains("emscripten") {
564            // FIXME: ideally, we would just have one config, and then have some mechanism of
565            // generating multiple variants of a test, one for each debugger (something like
566            // debuginfo revisions). But for now, we just create three configs.
567            configs.extend([
568                Arc::new(Config { debugger: Some(Debugger::Cdb), ..config.as_ref().clone() }),
569                Arc::new(Config { debugger: Some(Debugger::Gdb), ..config.as_ref().clone() }),
570                Arc::new(Config { debugger: Some(Debugger::Lldb), ..config.as_ref().clone() }),
571            ]);
572
573            // FIXME: this should ideally happen somewhere else..
574            if config.target.contains("android") {
575                println!("{} debug-info test uses tcp 5039 port. please reserve it", config.target);
576
577                // android debug-info test uses remote debugger so, we test 1 thread
578                // at once as they're all sharing the same TCP port to communicate
579                // over.
580                //
581                // we should figure out how to lift this restriction! (run them all
582                // on different ports allocated dynamically).
583                //
584                // SAFETY: at this point we are still single-threaded.
585                unsafe { env::set_var("RUST_TEST_THREADS", "1") };
586            }
587        }
588    } else {
589        configs.push(config.clone());
590    };
591
592    // Discover all of the tests in the test suite directory, and build a `CollectedTest`
593    // structure for each test (or each revision of a multi-revision test).
594    let mut tests = Vec::new();
595    for c in configs {
596        tests.extend(collect_and_make_tests(c));
597    }
598
599    tests.sort_by(|a, b| Ord::cmp(&a.desc.name, &b.desc.name));
600
601    // Delegate to the executor to filter and run the big list of test structures
602    // created during test discovery. When the executor decides to run a test,
603    // it will return control to the rest of compiletest by calling `runtest::run`.
604    let ok = executor::run_tests(&config, tests);
605
606    // Check the outcome reported by the executor.
607    if !ok {
608        // We want to report that the tests failed, but we also want to give
609        // some indication of just what tests we were running. Especially on
610        // CI, where there can be cross-compiled tests for a lot of
611        // architectures, without this critical information it can be quite
612        // easy to miss which tests failed, and as such fail to reproduce
613        // the failure locally.
614
615        let mut msg = String::from("Some tests failed in compiletest");
616        write!(msg, " suite={}", config.suite).unwrap();
617
618        if let Some(compare_mode) = config.compare_mode.as_ref() {
619            write!(msg, " compare_mode={}", compare_mode).unwrap();
620        }
621
622        if let Some(pass_mode) = config.force_pass_mode.as_ref() {
623            write!(msg, " pass_mode={}", pass_mode).unwrap();
624        }
625
626        write!(msg, " mode={}", config.mode).unwrap();
627        write!(msg, " host={}", config.host).unwrap();
628        write!(msg, " target={}", config.target).unwrap();
629
630        println!("{msg}");
631
632        std::process::exit(1);
633    }
634}
635
636/// Read-only context data used during test collection.
637struct TestCollectorCx {
638    config: Arc<Config>,
639    cache: DirectivesCache,
640    common_inputs_stamp: Stamp,
641    modified_tests: Vec<Utf8PathBuf>,
642}
643
644/// Mutable state used during test collection.
645struct TestCollector {
646    tests: Vec<CollectedTest>,
647    found_path_stems: HashSet<Utf8PathBuf>,
648    poisoned: bool,
649}
650
651impl TestCollector {
652    fn new() -> Self {
653        TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false }
654    }
655
656    fn merge(&mut self, mut other: Self) {
657        self.tests.append(&mut other.tests);
658        self.found_path_stems.extend(other.found_path_stems);
659        self.poisoned |= other.poisoned;
660    }
661}
662
663/// Creates test structures for every test/revision in the test suite directory.
664///
665/// This always inspects _all_ test files in the suite (e.g. all 17k+ ui tests),
666/// regardless of whether any filters/tests were specified on the command-line,
667/// because filtering is handled later by code that was copied from libtest.
668///
669/// FIXME(Zalathar): Now that we no longer rely on libtest, try to overhaul
670/// test discovery to take into account the filters/tests specified on the
671/// command-line, instead of having to enumerate everything.
672fn collect_and_make_tests(config: Arc<Config>) -> Vec<CollectedTest> {
673    debug!("making tests from {}", config.src_test_suite_root);
674    let common_inputs_stamp = common_inputs_stamp(&config);
675    let modified_tests =
676        modified_tests(&config, &config.src_test_suite_root).unwrap_or_else(|err| {
677            fatal!("modified_tests: {}: {err}", config.src_test_suite_root);
678        });
679    let cache = DirectivesCache::load(&config);
680
681    let cx = TestCollectorCx { config, cache, common_inputs_stamp, modified_tests };
682    let collector = collect_tests_from_dir(&cx, &cx.config.src_test_suite_root, Utf8Path::new(""))
683        .unwrap_or_else(|reason| {
684            panic!("Could not read tests from {}: {reason}", cx.config.src_test_suite_root)
685        });
686
687    let TestCollector { tests, found_path_stems, poisoned } = collector;
688
689    if poisoned {
690        eprintln!();
691        panic!("there are errors in tests");
692    }
693
694    check_for_overlapping_test_paths(&found_path_stems);
695
696    tests
697}
698
699/// Returns the most recent last-modified timestamp from among the input files
700/// that are considered relevant to all tests (e.g. the compiler, std, and
701/// compiletest itself).
702///
703/// (Some of these inputs aren't actually relevant to _all_ tests, but they are
704/// common to some subset of tests, and are hopefully unlikely to be modified
705/// while working on other tests.)
706fn common_inputs_stamp(config: &Config) -> Stamp {
707    let src_root = &config.src_root;
708
709    let mut stamp = Stamp::from_path(&config.rustc_path);
710
711    // Relevant pretty printer files
712    let pretty_printer_files = [
713        "src/etc/rust_types.py",
714        "src/etc/gdb_load_rust_pretty_printers.py",
715        "src/etc/gdb_lookup.py",
716        "src/etc/gdb_providers.py",
717        "src/etc/lldb_batchmode",
718        "src/etc/lldb_lookup.py",
719        "src/etc/lldb_providers.py",
720    ];
721    for file in &pretty_printer_files {
722        let path = src_root.join(file);
723        stamp.add_path(&path);
724    }
725
726    stamp.add_dir(&src_root.join("src/etc/natvis"));
727
728    stamp.add_dir(&config.target_run_lib_path);
729
730    if let Some(ref rustdoc_path) = config.rustdoc_path {
731        stamp.add_path(&rustdoc_path);
732        stamp.add_path(&src_root.join("src/etc/htmldocck.py"));
733    }
734
735    // Re-run coverage tests if the `coverage-dump` tool was modified,
736    // because its output format might have changed.
737    if let Some(coverage_dump_path) = &config.coverage_dump_path {
738        stamp.add_path(coverage_dump_path)
739    }
740
741    stamp.add_dir(&src_root.join("src/tools/run-make-support"));
742
743    // Compiletest itself.
744    stamp.add_dir(&src_root.join("src/tools/compiletest"));
745
746    stamp
747}
748
749/// Returns a list of modified/untracked test files that should be run when
750/// the `--only-modified` flag is in use.
751///
752/// (Might be inaccurate in some cases.)
753fn modified_tests(config: &Config, dir: &Utf8Path) -> Result<Vec<Utf8PathBuf>, String> {
754    // If `--only-modified` wasn't passed, the list of modified tests won't be
755    // used for anything, so avoid some work and just return an empty list.
756    if !config.only_modified {
757        return Ok(vec![]);
758    }
759
760    let files = get_git_modified_files(
761        &config.git_config(),
762        Some(dir.as_std_path()),
763        &vec!["rs", "stderr", "fixed"],
764    )?;
765    // Add new test cases to the list, it will be convenient in daily development.
766    let untracked_files = get_git_untracked_files(Some(dir.as_std_path()))?.unwrap_or(vec![]);
767
768    let all_paths = [&files[..], &untracked_files[..]].concat();
769    let full_paths = {
770        let mut full_paths: Vec<Utf8PathBuf> = all_paths
771            .into_iter()
772            .map(|f| Utf8PathBuf::from(f).with_extension("").with_extension("rs"))
773            .filter_map(
774                |f| if Utf8Path::new(&f).exists() { f.canonicalize_utf8().ok() } else { None },
775            )
776            .collect();
777        full_paths.dedup();
778        full_paths.sort_unstable();
779        full_paths
780    };
781    Ok(full_paths)
782}
783
784/// Recursively scans a directory to find test files and create test structures
785/// that will be handed over to the executor.
786fn collect_tests_from_dir(
787    cx: &TestCollectorCx,
788    dir: &Utf8Path,
789    relative_dir_path: &Utf8Path,
790) -> io::Result<TestCollector> {
791    // Ignore directories that contain a file named `compiletest-ignore-dir`.
792    if dir.join("compiletest-ignore-dir").exists() {
793        return Ok(TestCollector::new());
794    }
795
796    let mut components = dir.components().rev();
797    if let Some(Utf8Component::Normal(last)) = components.next()
798        && let Some(("assembly" | "codegen", backend)) = last.split_once('-')
799        && let Some(Utf8Component::Normal(parent)) = components.next()
800        && parent == "tests"
801        && let Ok(backend) = backend.parse::<CodegenBackend>()
802        && backend != cx.config.default_codegen_backend
803    {
804        // We ignore asm tests which don't match the current codegen backend.
805        warning!(
806            "Ignoring tests in `{dir}` because they don't match the configured codegen \
807             backend (`{}`)",
808            cx.config.default_codegen_backend.as_str(),
809        );
810        return Ok(TestCollector::new());
811    }
812
813    // For run-make tests, a "test file" is actually a directory that contains an `rmake.rs`.
814    if cx.config.mode == TestMode::RunMake {
815        let mut collector = TestCollector::new();
816        if dir.join("rmake.rs").exists() {
817            let paths = TestPaths {
818                file: dir.to_path_buf(),
819                relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
820            };
821            make_test(cx, &mut collector, &paths);
822            // This directory is a test, so don't try to find other tests inside it.
823            return Ok(collector);
824        }
825    }
826
827    // If we find a test foo/bar.rs, we have to build the
828    // output directory `$build/foo` so we can write
829    // `$build/foo/bar` into it. We do this *now* in this
830    // sequential loop because otherwise, if we do it in the
831    // tests themselves, they race for the privilege of
832    // creating the directories and sometimes fail randomly.
833    let build_dir = output_relative_path(&cx.config, relative_dir_path);
834    fs::create_dir_all(&build_dir).unwrap();
835
836    // Add each `.rs` file as a test, and recurse further on any
837    // subdirectories we find, except for `auxiliary` directories.
838    // FIXME: this walks full tests tree, even if we have something to ignore
839    // use walkdir/ignore like in tidy?
840    fs::read_dir(dir.as_std_path())?
841        .par_bridge()
842        .map(|file| {
843            let mut collector = TestCollector::new();
844            let file = file?;
845            let file_path = Utf8PathBuf::try_from(file.path()).unwrap();
846            let file_name = file_path.file_name().unwrap();
847
848            if is_test(file_name)
849                && (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
850            {
851                // We found a test file, so create the corresponding test structures.
852                debug!(%file_path, "found test file");
853
854                // Record the stem of the test file, to check for overlaps later.
855                let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
856                collector.found_path_stems.insert(rel_test_path);
857
858                let paths =
859                    TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
860                make_test(cx, &mut collector, &paths);
861            } else if file_path.is_dir() {
862                // Recurse to find more tests in a subdirectory.
863                let relative_file_path = relative_dir_path.join(file_name);
864                if file_name != "auxiliary" {
865                    debug!(%file_path, "found directory");
866                    collector.merge(collect_tests_from_dir(cx, &file_path, &relative_file_path)?);
867                }
868            } else {
869                debug!(%file_path, "found other file/directory");
870            }
871            Ok(collector)
872        })
873        .reduce(
874            || Ok(TestCollector::new()),
875            |a, b| {
876                let mut a = a?;
877                a.merge(b?);
878                Ok(a)
879            },
880        )
881}
882
883/// Returns true if `file_name` looks like a proper test file name.
884fn is_test(file_name: &str) -> bool {
885    if !file_name.ends_with(".rs") {
886        return false;
887    }
888
889    // `.`, `#`, and `~` are common temp-file prefixes.
890    let invalid_prefixes = &[".", "#", "~"];
891    !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
892}
893
894/// For a single test file, creates one or more test structures (one per revision) that can be
895/// handed over to the executor to run, possibly in parallel.
896fn make_test(cx: &TestCollectorCx, collector: &mut TestCollector, testpaths: &TestPaths) {
897    // For run-make tests, each "test file" is actually a _directory_ containing an `rmake.rs`. But
898    // for the purposes of directive parsing, we want to look at that recipe file, not the directory
899    // itself.
900    let test_path = if cx.config.mode == TestMode::RunMake {
901        testpaths.file.join("rmake.rs")
902    } else {
903        testpaths.file.clone()
904    };
905
906    // Scan the test file to discover its revisions, if any.
907    let file_contents =
908        fs::read_to_string(&test_path).expect("reading test file for directives should succeed");
909    let file_directives = FileDirectives::from_file_contents(&test_path, &file_contents);
910
911    if let Err(message) = directives::do_early_directives_check(cx.config.mode, &file_directives) {
912        // FIXME(Zalathar): Overhaul compiletest error handling so that we
913        // don't have to resort to ad-hoc panics everywhere.
914        panic!("directives check failed:\n{message}");
915    }
916    let early_props = EarlyProps::from_file_directives(&cx.config, &file_directives);
917
918    // Normally we create one structure per revision, with two exceptions:
919    // - If a test doesn't use revisions, create a dummy revision (None) so that
920    //   the test can still run.
921    // - Incremental tests inherently can't run their revisions in parallel, so
922    //   we treat them like non-revisioned tests here. Incremental revisions are
923    //   handled internally by `runtest::run` instead.
924    let revisions = if early_props.revisions.is_empty() || cx.config.mode == TestMode::Incremental {
925        vec![None]
926    } else {
927        early_props.revisions.iter().map(|r| Some(r.as_str())).collect()
928    };
929
930    // For each revision (or the sole dummy revision), create and append a
931    // `CollectedTest` that can be handed over to the test executor.
932    collector.tests.extend(revisions.into_iter().map(|revision| {
933        // Create a test name and description to hand over to the executor.
934        let (test_name, filterable_path) =
935            make_test_name_and_filterable_path(&cx.config, testpaths, revision);
936
937        // While scanning for ignore/only/needs directives, also collect aux
938        // paths for up-to-date checking.
939        let mut aux_props = AuxProps::default();
940
941        // Create a description struct for the test/revision.
942        // This is where `ignore-*`/`only-*`/`needs-*` directives are handled,
943        // because they historically needed to set the libtest ignored flag.
944        let mut desc = make_test_description(
945            &cx.config,
946            &cx.cache,
947            test_name,
948            &test_path,
949            &filterable_path,
950            &file_directives,
951            revision,
952            &mut collector.poisoned,
953            &mut aux_props,
954        );
955
956        // If a test's inputs haven't changed since the last time it ran,
957        // mark it as ignored so that the executor will skip it.
958        if !desc.is_ignored()
959            && !cx.config.force_rerun
960            && is_up_to_date(cx, testpaths, &aux_props, revision)
961        {
962            // Keep this in sync with the "up-to-date" message detected by bootstrap.
963            // FIXME(Zalathar): Now that we are no longer tied to libtest, we could
964            // find a less fragile way to communicate this status to bootstrap.
965            desc.ignore_message = Some("up-to-date".into());
966        }
967
968        let config = Arc::clone(&cx.config);
969        let testpaths = testpaths.clone();
970        let revision = revision.map(str::to_owned);
971
972        CollectedTest { desc, config, testpaths, revision }
973    }));
974}
975
976/// The path of the `stamp` file that gets created or updated whenever a
977/// particular test completes successfully.
978fn stamp_file_path(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> Utf8PathBuf {
979    output_base_dir(config, testpaths, revision).join("stamp")
980}
981
982/// Returns a list of files that, if modified, would cause this test to no
983/// longer be up-to-date.
984///
985/// (Might be inaccurate in some cases.)
986fn files_related_to_test(
987    config: &Config,
988    testpaths: &TestPaths,
989    aux_props: &AuxProps,
990    revision: Option<&str>,
991) -> Vec<Utf8PathBuf> {
992    let mut related = vec![];
993
994    if testpaths.file.is_dir() {
995        // run-make tests use their individual directory
996        for entry in WalkDir::new(&testpaths.file) {
997            let path = entry.unwrap().into_path();
998            if path.is_file() {
999                related.push(Utf8PathBuf::try_from(path).unwrap());
1000            }
1001        }
1002    } else {
1003        related.push(testpaths.file.clone());
1004    }
1005
1006    for aux in aux_props.all_aux_path_strings() {
1007        // FIXME(Zalathar): Perform all `auxiliary` path resolution in one place.
1008        // FIXME(Zalathar): This only finds auxiliary files used _directly_ by
1009        // the test file; if a transitive auxiliary is modified, the test might
1010        // be treated as "up-to-date" even though it should run.
1011        let path = testpaths.file.parent().unwrap().join("auxiliary").join(aux);
1012        related.push(path);
1013    }
1014
1015    // UI test files.
1016    for extension in UI_EXTENSIONS {
1017        let path = expected_output_path(testpaths, revision, &config.compare_mode, extension);
1018        related.push(path);
1019    }
1020
1021    // `minicore.rs` test auxiliary: we need to make sure tests get rerun if this changes.
1022    related.push(config.src_root.join("tests").join("auxiliary").join("minicore.rs"));
1023
1024    related
1025}
1026
1027/// Checks whether a particular test/revision is "up-to-date", meaning that no
1028/// relevant files/settings have changed since the last time the test succeeded.
1029///
1030/// (This is not very reliable in some circumstances, so the `--force-rerun`
1031/// flag can be used to ignore up-to-date checking and always re-run tests.)
1032fn is_up_to_date(
1033    cx: &TestCollectorCx,
1034    testpaths: &TestPaths,
1035    aux_props: &AuxProps,
1036    revision: Option<&str>,
1037) -> bool {
1038    let stamp_file_path = stamp_file_path(&cx.config, testpaths, revision);
1039    // Check the config hash inside the stamp file.
1040    let contents = match fs::read_to_string(&stamp_file_path) {
1041        Ok(f) => f,
1042        Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"),
1043        // The test hasn't succeeded yet, so it is not up-to-date.
1044        Err(_) => return false,
1045    };
1046    let expected_hash = runtest::compute_stamp_hash(&cx.config);
1047    if contents != expected_hash {
1048        // Some part of compiletest configuration has changed since the test
1049        // last succeeded, so it is not up-to-date.
1050        return false;
1051    }
1052
1053    // Check the timestamp of the stamp file against the last modified time
1054    // of all files known to be relevant to the test.
1055    let mut inputs_stamp = cx.common_inputs_stamp.clone();
1056    for path in files_related_to_test(&cx.config, testpaths, aux_props, revision) {
1057        inputs_stamp.add_path(&path);
1058    }
1059
1060    // If no relevant files have been modified since the stamp file was last
1061    // written, the test is up-to-date.
1062    inputs_stamp < Stamp::from_path(&stamp_file_path)
1063}
1064
1065/// The maximum of a set of file-modified timestamps.
1066#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
1067struct Stamp {
1068    time: SystemTime,
1069}
1070
1071impl Stamp {
1072    /// Creates a timestamp holding the last-modified time of the specified file.
1073    fn from_path(path: &Utf8Path) -> Self {
1074        let mut stamp = Stamp { time: SystemTime::UNIX_EPOCH };
1075        stamp.add_path(path);
1076        stamp
1077    }
1078
1079    /// Updates this timestamp to the last-modified time of the specified file,
1080    /// if it is later than the currently-stored timestamp.
1081    fn add_path(&mut self, path: &Utf8Path) {
1082        let modified = fs::metadata(path.as_std_path())
1083            .and_then(|metadata| metadata.modified())
1084            .unwrap_or(SystemTime::UNIX_EPOCH);
1085        self.time = self.time.max(modified);
1086    }
1087
1088    /// Updates this timestamp to the most recent last-modified time of all files
1089    /// recursively contained in the given directory, if it is later than the
1090    /// currently-stored timestamp.
1091    fn add_dir(&mut self, path: &Utf8Path) {
1092        let path = path.as_std_path();
1093        for entry in WalkDir::new(path) {
1094            let entry = entry.unwrap();
1095            if entry.file_type().is_file() {
1096                let modified = entry
1097                    .metadata()
1098                    .ok()
1099                    .and_then(|metadata| metadata.modified().ok())
1100                    .unwrap_or(SystemTime::UNIX_EPOCH);
1101                self.time = self.time.max(modified);
1102            }
1103        }
1104    }
1105}
1106
1107/// Creates a name for this test/revision that can be handed over to the executor.
1108fn make_test_name_and_filterable_path(
1109    config: &Config,
1110    testpaths: &TestPaths,
1111    revision: Option<&str>,
1112) -> (String, Utf8PathBuf) {
1113    // Print the name of the file, relative to the sources root.
1114    let path = testpaths.file.strip_prefix(&config.src_root).unwrap();
1115    let debugger = match config.debugger {
1116        Some(d) => format!("-{}", d),
1117        None => String::new(),
1118    };
1119    let mode_suffix = match config.compare_mode {
1120        Some(ref mode) => format!(" ({})", mode.to_str()),
1121        None => String::new(),
1122    };
1123
1124    let name = format!(
1125        "[{}{}{}] {}{}",
1126        config.mode,
1127        debugger,
1128        mode_suffix,
1129        path,
1130        revision.map_or("".to_string(), |rev| format!("#{}", rev))
1131    );
1132
1133    // `path` is the full path from the repo root like, `tests/ui/foo/bar.rs`.
1134    // Filtering is applied without the `tests/ui/` part, so strip that off.
1135    // First strip off "tests" to make sure we don't have some unexpected path.
1136    let mut filterable_path = path.strip_prefix("tests").unwrap().to_owned();
1137    // Now strip off e.g. "ui" or "run-make" component.
1138    filterable_path = filterable_path.components().skip(1).collect();
1139
1140    (name, filterable_path)
1141}
1142
1143/// Checks that test discovery didn't find any tests whose name stem is a prefix
1144/// of some other tests's name.
1145///
1146/// For example, suppose the test suite contains these two test files:
1147/// - `tests/rustdoc-html/primitive.rs`
1148/// - `tests/rustdoc-html/primitive/no_std.rs`
1149///
1150/// The test runner might put the output from those tests in these directories:
1151/// - `$build/test/rustdoc/primitive/`
1152/// - `$build/test/rustdoc/primitive/no_std/`
1153///
1154/// Because one output path is a subdirectory of the other, the two tests might
1155/// interfere with each other in unwanted ways, especially if the test runner
1156/// decides to delete test output directories to clean them between runs.
1157/// To avoid problems, we forbid test names from overlapping in this way.
1158///
1159/// See <https://github.com/rust-lang/rust/pull/109509> for more context.
1160fn check_for_overlapping_test_paths(found_path_stems: &HashSet<Utf8PathBuf>) {
1161    let mut collisions = Vec::new();
1162    for path in found_path_stems {
1163        for ancestor in path.ancestors().skip(1) {
1164            if found_path_stems.contains(ancestor) {
1165                collisions.push((path, ancestor));
1166            }
1167        }
1168    }
1169    if !collisions.is_empty() {
1170        collisions.sort();
1171        let collisions: String = collisions
1172            .into_iter()
1173            .map(|(path, check_parent)| format!("test {path} clashes with {check_parent}\n"))
1174            .collect();
1175        panic!(
1176            "{collisions}\n\
1177            Tests cannot have overlapping names. Make sure they use unique prefixes."
1178        );
1179    }
1180}
1181
1182fn early_config_check(config: &Config) {
1183    if !config.profiler_runtime && config.mode == TestMode::CoverageRun {
1184        let actioned = if config.bless { "blessed" } else { "checked" };
1185        warning!("profiler runtime is not available, so `.coverage` files won't be {actioned}");
1186        help!("try setting `profiler = true` in the `[build]` section of `bootstrap.toml`");
1187    }
1188
1189    // `RUST_TEST_NOCAPTURE` is a libtest env var, but we don't callout to libtest.
1190    if env::var("RUST_TEST_NOCAPTURE").is_ok() {
1191        warning!("`RUST_TEST_NOCAPTURE` is not supported; use the `--no-capture` flag instead");
1192    }
1193}
compiletest/lib.rs

compiletest/
lib.rs
