bootstrap/utils/
tarball.rs

1//! Facilitates the management and generation of tarballs.
2//!
3//! Tarballs efficiently hold Rust compiler build artifacts and
4//! capture a snapshot of each bootstrap stage.
5//! In uplifting, a tarball from Stage N captures essential components
6//! to assemble Stage N + 1 compiler.
7
8use std::path::{Path, PathBuf};
9
10use crate::core::build_steps::dist::distdir;
11use crate::core::builder::{Builder, Kind};
12use crate::core::config::BUILDER_CONFIG_FILENAME;
13use crate::utils::exec::BootstrapCommand;
14use crate::utils::helpers::{move_file, t};
15use crate::utils::{channel, helpers};
16
17#[derive(Copy, Clone)]
18pub(crate) enum OverlayKind {
19    Rust,
20    Llvm,
21    Cargo,
22    Clippy,
23    Miri,
24    Rustfmt,
25    RustAnalyzer,
26    RustcCodegenCranelift,
27    LlvmBitcodeLinker,
28}
29
30impl OverlayKind {
31    fn legal_and_readme(&self) -> &[&str] {
32        match self {
33            OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
34            OverlayKind::Llvm => {
35                &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
36            }
37            OverlayKind::Cargo => &[
38                "src/tools/cargo/README.md",
39                "src/tools/cargo/LICENSE-MIT",
40                "src/tools/cargo/LICENSE-APACHE",
41                "src/tools/cargo/LICENSE-THIRD-PARTY",
42            ],
43            OverlayKind::Clippy => &[
44                "src/tools/clippy/README.md",
45                "src/tools/clippy/LICENSE-APACHE",
46                "src/tools/clippy/LICENSE-MIT",
47            ],
48            OverlayKind::Miri => &[
49                "src/tools/miri/README.md",
50                "src/tools/miri/LICENSE-APACHE",
51                "src/tools/miri/LICENSE-MIT",
52            ],
53            OverlayKind::Rustfmt => &[
54                "src/tools/rustfmt/README.md",
55                "src/tools/rustfmt/LICENSE-APACHE",
56                "src/tools/rustfmt/LICENSE-MIT",
57            ],
58            OverlayKind::RustAnalyzer => &[
59                "src/tools/rust-analyzer/README.md",
60                "src/tools/rust-analyzer/LICENSE-APACHE",
61                "src/tools/rust-analyzer/LICENSE-MIT",
62            ],
63            OverlayKind::RustcCodegenCranelift => &[
64                "compiler/rustc_codegen_cranelift/Readme.md",
65                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
66                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
67            ],
68            OverlayKind::LlvmBitcodeLinker => &[
69                "COPYRIGHT",
70                "LICENSE-APACHE",
71                "LICENSE-MIT",
72                "src/tools/llvm-bitcode-linker/README.md",
73            ],
74        }
75    }
76
77    fn version(&self, builder: &Builder<'_>) -> String {
78        match self {
79            OverlayKind::Rust => builder.rust_version(),
80            OverlayKind::Llvm => builder.rust_version(),
81            OverlayKind::Cargo => {
82                builder.cargo_info.version(builder, &builder.release_num("cargo"))
83            }
84            OverlayKind::Clippy => {
85                builder.clippy_info.version(builder, &builder.release_num("clippy"))
86            }
87            OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
88            OverlayKind::Rustfmt => {
89                builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
90            }
91            OverlayKind::RustAnalyzer => builder
92                .rust_analyzer_info
93                .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
94            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
95            OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
96        }
97    }
98}
99
100pub(crate) struct Tarball<'a> {
101    builder: &'a Builder<'a>,
102
103    pkgname: String,
104    component: String,
105    target: Option<String>,
106    product_name: String,
107    overlay: OverlayKind,
108
109    temp_dir: PathBuf,
110    image_dir: PathBuf,
111    overlay_dir: PathBuf,
112    bulk_dirs: Vec<PathBuf>,
113
114    include_target_in_component_name: bool,
115    is_preview: bool,
116    permit_symlinks: bool,
117}
118
119impl<'a> Tarball<'a> {
120    pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
121        Self::new_inner(builder, component, Some(target.into()))
122    }
123
124    pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
125        Self::new_inner(builder, component, None)
126    }
127
128    fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
129        let pkgname = crate::core::build_steps::dist::pkgname(builder, component);
130
131        let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
132        if let Some(target) = &target {
133            temp_dir = temp_dir.join(target);
134        }
135        let _ = std::fs::remove_dir_all(&temp_dir);
136
137        let image_dir = temp_dir.join("image");
138        let overlay_dir = temp_dir.join("overlay");
139
140        Self {
141            builder,
142
143            pkgname,
144            component: component.into(),
145            target,
146            product_name: "Rust".into(),
147            overlay: OverlayKind::Rust,
148
149            temp_dir,
150            image_dir,
151            overlay_dir,
152            bulk_dirs: Vec::new(),
153
154            include_target_in_component_name: false,
155            is_preview: false,
156            permit_symlinks: false,
157        }
158    }
159
160    pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
161        self.overlay = overlay;
162    }
163
164    pub(crate) fn set_product_name(&mut self, name: &str) {
165        self.product_name = name.into();
166    }
167
168    pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
169        self.include_target_in_component_name = include;
170    }
171
172    pub(crate) fn is_preview(&mut self, is: bool) {
173        self.is_preview = is;
174    }
175
176    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
177        self.permit_symlinks = flag;
178    }
179
180    pub(crate) fn image_dir(&self) -> &Path {
181        t!(std::fs::create_dir_all(&self.image_dir));
182        &self.image_dir
183    }
184
185    pub(crate) fn add_file(&self, src: impl AsRef<Path>, destdir: impl AsRef<Path>, perms: u32) {
186        // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
187        // uses the base directory as the destination directory.
188        let destdir = if destdir.as_ref() == Path::new(".") {
189            self.image_dir.clone()
190        } else {
191            self.image_dir.join(destdir.as_ref())
192        };
193
194        t!(std::fs::create_dir_all(&destdir));
195        self.builder.install(src.as_ref(), &destdir, perms);
196    }
197
198    pub(crate) fn add_renamed_file(
199        &self,
200        src: impl AsRef<Path>,
201        destdir: impl AsRef<Path>,
202        new_name: &str,
203    ) {
204        let destdir = self.image_dir.join(destdir.as_ref());
205        t!(std::fs::create_dir_all(&destdir));
206        self.builder.copy_link(src.as_ref(), &destdir.join(new_name));
207    }
208
209    pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
210        for file in self.overlay.legal_and_readme() {
211            self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644);
212        }
213    }
214
215    pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
216        let dest = self.image_dir.join(dest.as_ref());
217
218        t!(std::fs::create_dir_all(&dest));
219        self.builder.cp_link_r(src.as_ref(), &dest);
220    }
221
222    pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
223        self.bulk_dirs.push(dest.as_ref().to_path_buf());
224        self.add_dir(src, dest);
225    }
226
227    pub(crate) fn generate(self) -> GeneratedTarball {
228        let mut component_name = self.component.clone();
229        if self.is_preview {
230            component_name.push_str("-preview");
231        }
232        if self.include_target_in_component_name {
233            component_name.push('-');
234            component_name.push_str(
235                self.target
236                    .as_ref()
237                    .expect("include_target_in_component_name used in a targetless tarball"),
238            );
239        }
240
241        self.run(|this, cmd| {
242            cmd.arg("generate")
243                .arg("--image-dir")
244                .arg(&this.image_dir)
245                .arg(format!("--component-name={component_name}"));
246
247            if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
248                let mut arg = dir.as_os_str().to_os_string();
249                for dir in dirs {
250                    arg.push(",");
251                    arg.push(dir);
252                }
253                cmd.arg("--bulk-dirs").arg(&arg);
254            }
255
256            this.non_bare_args(cmd);
257        })
258    }
259
260    pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
261        let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
262        for tarball in &tarballs[1..] {
263            input_tarballs.push(",");
264            input_tarballs.push(&tarball.path);
265        }
266
267        self.run(|this, cmd| {
268            cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
269            this.non_bare_args(cmd);
270        })
271    }
272
273    pub(crate) fn bare(self) -> GeneratedTarball {
274        // Bare tarballs should have the top level directory match the package
275        // name, not "image". We rename the image directory just before passing
276        // into rust-installer.
277        let dest = self.temp_dir.join(self.package_name());
278        t!(move_file(&self.image_dir, &dest));
279
280        self.run(|this, cmd| {
281            let distdir = distdir(this.builder);
282            t!(std::fs::create_dir_all(&distdir));
283            cmd.arg("tarball")
284                .arg("--input")
285                .arg(&dest)
286                .arg("--output")
287                .arg(distdir.join(this.package_name()));
288        })
289    }
290
291    fn package_name(&self) -> String {
292        if let Some(target) = &self.target {
293            format!("{}-{}", self.pkgname, target)
294        } else {
295            self.pkgname.clone()
296        }
297    }
298
299    fn non_bare_args(&self, cmd: &mut BootstrapCommand) {
300        cmd.arg("--rel-manifest-dir=rustlib")
301            .arg("--legacy-manifest-dirs=rustlib,cargo")
302            .arg(format!("--product-name={}", self.product_name))
303            .arg(format!("--success-message={} installed.", self.component))
304            .arg(format!("--package-name={}", self.package_name()))
305            .arg("--non-installed-overlay")
306            .arg(&self.overlay_dir)
307            .arg("--output-dir")
308            .arg(distdir(self.builder));
309    }
310
311    fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball {
312        t!(std::fs::create_dir_all(&self.overlay_dir));
313        self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
314        if let Some(info) = self.builder.rust_info().info() {
315            channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
316            channel::write_commit_info_file(&self.overlay_dir, info);
317        }
318
319        // Add config file if present.
320        if let Some(config) = &self.builder.config.config {
321            self.add_renamed_file(config, &self.overlay_dir, BUILDER_CONFIG_FILENAME);
322        }
323
324        for file in self.overlay.legal_and_readme() {
325            self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644);
326        }
327
328        let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller);
329
330        let package_name = self.package_name();
331        self.builder.info(&format!("Dist {package_name}"));
332        let _time = crate::utils::helpers::timeit(self.builder);
333
334        build_cli(&self, &mut cmd);
335        cmd.arg("--work-dir").arg(&self.temp_dir);
336        if let Some(formats) = &self.builder.config.dist_compression_formats {
337            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
338            cmd.arg("--compression-formats").arg(formats.join(","));
339        }
340
341        // For `x install` tarball files aren't needed, so we can speed up the process by not producing them.
342        let compression_profile = if self.builder.kind == Kind::Install {
343            self.builder.verbose(|| {
344                println!("Forcing dist.compression-profile = 'no-op' for `x install`.")
345            });
346            // "no-op" indicates that the rust-installer won't produce compressed tarball sources.
347            "no-op"
348        } else {
349            assert!(
350                self.builder.config.dist_compression_profile != "no-op",
351                "dist.compression-profile = 'no-op' can only be used for `x install`"
352            );
353
354            &self.builder.config.dist_compression_profile
355        };
356
357        cmd.args(["--compression-profile", compression_profile]);
358
359        // We want to use a pinned modification time for files in the archive
360        // to achieve better reproducibility. However, using the same mtime for all
361        // releases is not ideal, because it can break e.g. Cargo mtime checking
362        // (https://github.com/rust-lang/rust/issues/125578).
363        // Therefore, we set mtime to the date of the latest commit (if we're managed
364        // by git). In this way, the archive will still be always the same for a given commit
365        // (achieving reproducibility), but it will also change between different commits and
366        // Rust versions, so that it won't break mtime-based caches.
367        //
368        // Note that this only overrides the mtime of files, not directories, due to the
369        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
370
371        // Get the UTC timestamp of the last git commit, if we're under git.
372        // We need to use UTC, so that anyone who tries to rebuild from the same commit
373        // gets the same timestamp.
374        if self.builder.rust_info().is_managed_git_subrepository() {
375            // %ct means committer date
376            let timestamp = helpers::git(Some(&self.builder.src))
377                .arg("log")
378                .arg("-1")
379                .arg("--format=%ct")
380                .run_capture_stdout(self.builder)
381                .stdout();
382            cmd.args(["--override-file-mtime", timestamp.trim()]);
383        }
384
385        cmd.run(self.builder);
386
387        // Ensure there are no symbolic links in the tarball. In particular,
388        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
389        let decompressed_output = self.temp_dir.join(&package_name);
390        if !self.builder.config.dry_run() && !self.permit_symlinks {
391            for entry in walkdir::WalkDir::new(&decompressed_output) {
392                let entry = t!(entry);
393                if entry.path_is_symlink() {
394                    panic!("generated a symlink in a tarball: {}", entry.path().display());
395                }
396            }
397        }
398
399        // Use either the first compression format defined, or "gz" as the default.
400        let ext = self
401            .builder
402            .config
403            .dist_compression_formats
404            .as_ref()
405            .and_then(|formats| formats.first())
406            .map(|s| s.as_str())
407            .unwrap_or("gz");
408
409        GeneratedTarball {
410            path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")),
411            decompressed_output,
412            work: self.temp_dir,
413        }
414    }
415}
416
417#[derive(Debug, Clone)]
418pub struct GeneratedTarball {
419    path: PathBuf,
420    decompressed_output: PathBuf,
421    work: PathBuf,
422}
423
424impl GeneratedTarball {
425    pub(crate) fn tarball(&self) -> &Path {
426        &self.path
427    }
428
429    pub(crate) fn decompressed_output(&self) -> &Path {
430        &self.decompressed_output
431    }
432
433    pub(crate) fn work_dir(&self) -> &Path {
434        &self.work
435    }
436}
bootstrap/utils/tarball.rs

bootstrap/utils/
tarball.rs
