1use crate::Result;
11use crate::generic_decompress::Decompressor;
12use anyhow::{Context, anyhow};
13use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
14
15use cap_std::io_lifetimes;
16use cap_std_ext::cap_std::fs::Dir;
17use cap_std_ext::cmdext::{CapStdExtCommandExt, CmdFds};
18use cap_std_ext::{cap_std, cap_tempfile};
19use containers_image_proxy::oci_spec::image as oci_image;
20use fn_error_context::context;
21use ostree::gio;
22use ostree::prelude::FileExt;
23use std::borrow::Cow;
24use std::collections::{BTreeMap, HashMap};
25use std::io::{BufWriter, Seek, Write};
26use std::path::Path;
27use std::process::Stdio;
28use std::sync::Arc;
29use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
30use tracing::instrument;
31
32const EXCLUDED_TOPLEVEL_PATHS: &[&str] = &["run", "tmp", "proc", "sys", "dev"];
35
36#[context("Copying entry")]
38pub(crate) fn copy_entry(
39 mut entry: tar::Entry<impl std::io::Read>,
40 dest: &mut tar::Builder<impl std::io::Write>,
41 path: Option<&Path>,
42) -> Result<()> {
43 let path = if let Some(path) = path {
45 path.to_owned()
46 } else {
47 (*entry.path()?).to_owned()
48 };
49 let mut header = entry.header().clone();
50 if let Some(headers) = entry.pax_extensions()? {
51 let mut extensions_to_keep = Vec::new();
57 for ext_res in headers {
58 let ext = ext_res?;
59 let key = ext.key()?;
60 if key != "path" && key != "linkpath" {
61 extensions_to_keep.push((key, ext.value_bytes()));
62 }
63 }
64 if !extensions_to_keep.is_empty() {
65 dest.append_pax_extensions(extensions_to_keep)?;
66 }
67 }
68
69 match entry.header().entry_type() {
73 tar::EntryType::Symlink => {
74 let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?;
75 let target: &Utf8Path = (&*target).try_into()?;
77 dest.append_link(&mut header, path, target)
78 }
79 tar::EntryType::Link => {
80 let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?;
81 let target: &Utf8Path = (&*target).try_into()?;
82 let target = remap_etc_path(target);
85 dest.append_link(&mut header, path, &*target)
86 }
87 _ => dest.append_data(&mut header, path, entry),
88 }
89 .map_err(Into::into)
90}
91
92#[derive(Debug, Default)]
94#[non_exhaustive]
95pub struct WriteTarOptions {
96 pub base: Option<String>,
98 pub selinux: bool,
101 pub allow_nonusr: bool,
103 pub retain_var: bool,
106}
107
108#[derive(Debug, Default)]
113pub struct WriteTarResult {
114 pub commit: String,
116 pub filtered: BTreeMap<String, u32>,
118}
119
120fn sepolicy_from_base(repo: &ostree::Repo, base: &str) -> Result<tempfile::TempDir> {
123 let cancellable = gio::Cancellable::NONE;
124 let policypath = "usr/etc/selinux";
125 let tempdir = tempfile::tempdir()?;
126 let (root, _) = repo.read_commit(base, cancellable)?;
127 let policyroot = root.resolve_relative_path(policypath);
128 if policyroot.query_exists(cancellable) {
129 let policydest = tempdir.path().join(policypath);
130 std::fs::create_dir_all(policydest.parent().unwrap())?;
131 let opts = ostree::RepoCheckoutAtOptions {
132 mode: ostree::RepoCheckoutMode::User,
133 subpath: Some(Path::new(policypath).to_owned()),
134 ..Default::default()
135 };
136 repo.checkout_at(Some(&opts), ostree::AT_FDCWD, policydest, base, cancellable)?;
137 }
138 Ok(tempdir)
139}
140
141#[derive(Debug, PartialEq, Eq)]
142enum NormalizedPathResult<'a> {
143 Filtered(&'a str),
144 Normal(Utf8PathBuf),
145}
146
147#[derive(Debug, Clone, PartialEq, Eq, Default)]
148pub(crate) struct TarImportConfig {
149 allow_nonusr: bool,
150 remap_factory_var: bool,
151}
152
153fn remap_etc_path(path: &Utf8Path) -> Cow<'_, Utf8Path> {
155 let mut components = path.components();
156 let Some(prefix) = components.next() else {
157 return Cow::Borrowed(path);
158 };
159 let (prefix, first) = if matches!(prefix, Utf8Component::CurDir | Utf8Component::RootDir) {
160 let Some(next) = components.next() else {
161 return Cow::Borrowed(path);
162 };
163 (Some(prefix), next)
164 } else {
165 (None, prefix)
166 };
167 if first.as_str() == "etc" {
168 let usr = Utf8Component::Normal("usr");
169 Cow::Owned(
170 prefix
171 .into_iter()
172 .chain([usr, first])
173 .chain(components)
174 .collect(),
175 )
176 } else {
177 Cow::Borrowed(path)
178 }
179}
180
181fn normalize_validate_path<'a>(
182 path: &'a Utf8Path,
183 config: &'_ TarImportConfig,
184) -> Result<NormalizedPathResult<'a>> {
185 let mut components = path
187 .components()
188 .map(|part| {
189 match part {
190 camino::Utf8Component::RootDir => Ok(camino::Utf8Component::CurDir),
192 camino::Utf8Component::Normal(_) | camino::Utf8Component::CurDir => Ok(part),
194 _ => Err(anyhow!("Invalid path: {}", path)),
196 }
197 })
198 .peekable();
199 let mut ret = Utf8PathBuf::new();
200 if let Some(Ok(camino::Utf8Component::Normal(_))) = components.peek() {
202 ret.push(camino::Utf8Component::CurDir);
203 }
204 let mut found_first = false;
205 let mut excluded = false;
206 for part in components {
207 let part = part?;
208 if excluded {
209 return Ok(NormalizedPathResult::Filtered(part.as_str()));
210 }
211 if !found_first {
212 if let Utf8Component::Normal(part) = part {
213 found_first = true;
214 match part {
215 "usr" => ret.push(part),
217 "etc" => {
219 ret.push("usr/etc");
220 }
221 "var" => {
222 if config.remap_factory_var {
224 ret.push("usr/share/factory/var");
225 } else {
226 ret.push(part)
227 }
228 }
229 o if EXCLUDED_TOPLEVEL_PATHS.contains(&o) => {
230 excluded = true;
233 ret.push(part)
234 }
235 _ if config.allow_nonusr => ret.push(part),
236 _ => {
237 return Ok(NormalizedPathResult::Filtered(part));
238 }
239 }
240 } else {
241 ret.push(part);
242 }
243 } else {
244 ret.push(part);
245 }
246 }
247
248 Ok(NormalizedPathResult::Normal(ret))
249}
250
251pub(crate) fn filter_tar(
261 src: impl std::io::Read,
262 dest: impl std::io::Write,
263 config: &TarImportConfig,
264 tmpdir: &Dir,
265) -> Result<BTreeMap<String, u32>> {
266 let src = std::io::BufReader::new(src);
267 let mut src = tar::Archive::new(src);
268 let dest = BufWriter::new(dest);
269 let mut dest = tar::Builder::new(dest);
270 let mut filtered = BTreeMap::new();
271
272 let ents = src.entries()?;
273
274 tracing::debug!("Filtering tar; config={config:?}");
275
276 let mut changed_sysroot_objects = HashMap::new();
278 let mut new_sysroot_link_targets = HashMap::<Utf8PathBuf, Utf8PathBuf>::new();
279
280 for entry in ents {
281 let mut entry = entry?;
282 let header = entry.header();
283 let path = entry.path()?;
284 let path: &Utf8Path = (&*path).try_into()?;
285 let path = path.strip_prefix("/").unwrap_or(path);
287
288 let is_modified = header.mtime().unwrap_or_default() > 0;
289 let is_regular = header.entry_type() == tar::EntryType::Regular;
290 if path.strip_prefix(crate::tar::REPO_PREFIX).is_ok() {
291 if is_modified && is_regular {
296 tracing::debug!("Processing modified sysroot file {path}");
297 let mut tmpf = cap_tempfile::TempFile::new_anonymous(tmpdir)
299 .map(BufWriter::new)
300 .context("Creating tmpfile")?;
301 let path = path.to_owned();
302 let header = header.clone();
303 std::io::copy(&mut entry, &mut tmpf)
304 .map_err(anyhow::Error::msg)
305 .context("Copying")?;
306 let mut tmpf = tmpf.into_inner()?;
307 tmpf.seek(std::io::SeekFrom::Start(0))?;
308 changed_sysroot_objects.insert(path, (header, tmpf));
310 continue;
311 }
312 } else if header.entry_type() == tar::EntryType::Link && is_modified {
313 let target = header
314 .link_name()?
315 .ok_or_else(|| anyhow!("Invalid empty hardlink"))?;
316 let target: &Utf8Path = (&*target).try_into()?;
317 let target = path.strip_prefix("/").unwrap_or(target);
319 if target.strip_prefix(crate::tar::REPO_PREFIX).is_ok() {
321 if let Some((mut header, data)) = changed_sysroot_objects.remove(target) {
323 tracing::debug!("Making {path} canonical for sysroot link {target}");
324 dest.append_data(&mut header, path, data)?;
326 new_sysroot_link_targets.insert(target.to_owned(), path.to_owned());
328 } else if let Some(real_target) = new_sysroot_link_targets.get(target) {
329 tracing::debug!("Relinking {path} to {real_target}");
330 let mut header = header.clone();
333 dest.append_link(&mut header, path, real_target)?;
334 } else {
335 tracing::debug!("Found unhandled modified link from {path} to {target}");
336 }
337 continue;
338 }
339 }
340
341 let normalized = match normalize_validate_path(path, config)? {
342 NormalizedPathResult::Filtered(path) => {
343 tracing::trace!("Filtered: {path}");
344 if let Some(v) = filtered.get_mut(path) {
345 *v += 1;
346 } else {
347 filtered.insert(path.to_string(), 1);
348 }
349 continue;
350 }
351 NormalizedPathResult::Normal(path) => path,
352 };
353
354 copy_entry(entry, &mut dest, Some(normalized.as_std_path()))?;
355 }
356 dest.into_inner()?.flush()?;
357 Ok(filtered)
358}
359
360#[context("Filtering tar stream")]
362async fn filter_tar_async(
363 src: impl AsyncRead + Send + 'static,
364 media_type: oci_image::MediaType,
365 mut dest: impl AsyncWrite + Send + Unpin,
366 config: &TarImportConfig,
367 repo_tmpdir: Dir,
368) -> Result<BTreeMap<String, u32>> {
369 let (tx_buf, mut rx_buf) = tokio::io::duplex(8192);
370 let src = Box::pin(src);
372 let config = config.clone();
373 let tar_transformer = crate::tokio_util::spawn_blocking_flatten(move || {
374 let src = tokio_util::io::SyncIoBridge::new(src);
375 let mut src = Decompressor::new(&media_type, src)?;
376 let dest = tokio_util::io::SyncIoBridge::new(tx_buf);
377
378 let r = filter_tar(&mut src, dest, &config, &repo_tmpdir);
379
380 src.finish()?;
381
382 Ok(r)
383 });
384 let copier = tokio::io::copy(&mut rx_buf, &mut dest);
385 let (r, v) = tokio::join!(tar_transformer, copier);
386 let _v: u64 = v?;
387 r?
388}
389
390#[allow(unsafe_code)] #[instrument(level = "debug", skip_all)]
393pub async fn write_tar(
394 repo: &ostree::Repo,
395 src: impl tokio::io::AsyncRead + Send + Unpin + 'static,
396 media_type: oci_image::MediaType,
397 refname: &str,
398 options: Option<WriteTarOptions>,
399) -> Result<WriteTarResult> {
400 let repo = repo.clone();
401 let options = options.unwrap_or_default();
402 let sepolicy = if options.selinux {
403 if let Some(base) = options.base {
404 Some(sepolicy_from_base(&repo, &base).context("tar: Preparing sepolicy")?)
405 } else {
406 None
407 }
408 } else {
409 None
410 };
411 let mut c = std::process::Command::new("ostree");
412 c.env_remove("G_MESSAGES_DEBUG");
418 let repofd = repo.dfd_as_file()?;
419 let repofd: Arc<io_lifetimes::OwnedFd> = Arc::new(repofd.into());
420 {
421 let c = c
422 .stdin(Stdio::piped())
423 .stdout(Stdio::piped())
424 .stderr(Stdio::piped())
425 .args(["commit"]);
426 let mut fds = CmdFds::new();
427 fds.take_fd_n(repofd.clone(), 3);
428 c.take_fds(fds);
429 c.arg("--repo=/proc/self/fd/3");
430 if let Some(sepolicy) = sepolicy.as_ref() {
431 c.arg("--selinux-policy");
432 c.arg(sepolicy.path());
433 }
434 c.arg(format!(
435 "--add-metadata-string=ostree.importer.version={}",
436 env!("CARGO_PKG_VERSION")
437 ));
438 c.args([
439 "--no-bindings",
440 "--tar-autocreate-parents",
441 "--tree=tar=/proc/self/fd/0",
442 "--branch",
443 refname,
444 ]);
445 }
446 let mut c = tokio::process::Command::from(c);
447 c.kill_on_drop(true);
448 let mut r = c.spawn()?;
449 tracing::trace!("Spawned ostree child process");
450 let child_stdin = r.stdin.take().unwrap();
452 let mut child_stdout = r.stdout.take().unwrap();
453 let mut child_stderr = r.stderr.take().unwrap();
454 let import_config = TarImportConfig {
456 allow_nonusr: options.allow_nonusr,
457 remap_factory_var: !options.retain_var,
458 };
459 let repo_tmpdir = Dir::reopen_dir(&repo.dfd_borrow())?
460 .open_dir("tmp")
461 .context("Getting repo tmpdir")?;
462 let filtered_result =
463 filter_tar_async(src, media_type, child_stdin, &import_config, repo_tmpdir);
464 let output_copier = async move {
465 let mut child_stdout_buf = String::new();
467 let mut child_stderr_buf = String::new();
468 let (_a, _b) = tokio::try_join!(
469 child_stdout.read_to_string(&mut child_stdout_buf),
470 child_stderr.read_to_string(&mut child_stderr_buf)
471 )?;
472 Ok::<_, anyhow::Error>((child_stdout_buf, child_stderr_buf))
473 };
474
475 let status = async move {
478 let status = r.wait().await?;
479 if !status.success() {
480 return Err(anyhow!("Failed to commit tar: {:?}", status));
481 }
482 anyhow::Ok(())
483 };
484 tracing::debug!("Waiting on child process");
485 let (filtered_result, child_stdout) =
486 match tokio::try_join!(status, filtered_result).context("Processing tar") {
487 Ok(((), filtered_result)) => {
488 let (child_stdout, _) = output_copier.await.context("Copying child output")?;
489 (filtered_result, child_stdout)
490 }
491 Err(e) => {
492 if let Ok((_, child_stderr)) = output_copier.await {
493 let child_stderr = child_stderr.trim();
495 Err(e.context(child_stderr.to_string()))?
496 } else {
497 Err(e)?
498 }
499 }
500 };
501 drop(sepolicy);
502
503 tracing::trace!("tar written successfully");
504 let s = child_stdout.trim();
506 Ok(WriteTarResult {
507 commit: s.to_string(),
508 filtered: filtered_result,
509 })
510}
511
512#[cfg(test)]
513mod tests {
514 use super::*;
515 use std::io::Cursor;
516
517 #[test]
518 fn test_remap_etc() {
519 let unchanged = ["", "foo", "/etcc/foo", "../etc/baz"];
521 for x in unchanged {
522 similar_asserts::assert_eq!(x, remap_etc_path(x.into()).as_str());
523 }
524 for (p, expected) in [
527 ("/etc/foo/../bar/baz", "/usr/etc/foo/../bar/baz"),
528 ("etc/foo//bar", "usr/etc/foo/bar"),
529 ("./etc/foo", "./usr/etc/foo"),
530 ("etc", "usr/etc"),
531 ] {
532 similar_asserts::assert_eq!(remap_etc_path(p.into()).as_str(), expected);
533 }
534 }
535
536 #[test]
537 fn test_normalize_path() {
538 let imp_default = &TarImportConfig {
539 allow_nonusr: false,
540 remap_factory_var: true,
541 };
542 let allow_nonusr = &TarImportConfig {
543 allow_nonusr: true,
544 remap_factory_var: true,
545 };
546 let composefs_and_new_ostree = &TarImportConfig {
547 allow_nonusr: true,
548 remap_factory_var: false,
549 };
550 let valid_all = &[
551 ("/usr/bin/blah", "./usr/bin/blah"),
552 ("usr/bin/blah", "./usr/bin/blah"),
553 ("usr///share/.//blah", "./usr/share/blah"),
554 ("var/lib/blah", "./usr/share/factory/var/lib/blah"),
555 ("./var/lib/blah", "./usr/share/factory/var/lib/blah"),
556 ("dev", "./dev"),
557 ("/proc", "./proc"),
558 ("./", "."),
559 ];
560 let valid_nonusr = &[("boot", "./boot"), ("opt/puppet/blah", "./opt/puppet/blah")];
561 for &(k, v) in valid_all {
562 let r = normalize_validate_path(k.into(), imp_default).unwrap();
563 let r2 = normalize_validate_path(k.into(), allow_nonusr).unwrap();
564 assert_eq!(r, r2);
565 match r {
566 NormalizedPathResult::Normal(r) => assert_eq!(r, v),
567 NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"),
568 }
569 }
570 for &(k, v) in valid_nonusr {
571 let strict = normalize_validate_path(k.into(), imp_default).unwrap();
572 assert!(
573 matches!(strict, NormalizedPathResult::Filtered(_)),
574 "Incorrect filter for {k}"
575 );
576 let nonusr = normalize_validate_path(k.into(), allow_nonusr).unwrap();
577 match nonusr {
578 NormalizedPathResult::Normal(r) => assert_eq!(r, v),
579 NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"),
580 }
581 }
582 let filtered = &["/run/blah", "/sys/foo", "/dev/somedev"];
583 for &k in filtered {
584 match normalize_validate_path(k.into(), imp_default).unwrap() {
585 NormalizedPathResult::Filtered(_) => {}
586 NormalizedPathResult::Normal(_) => {
587 panic!("{k} should be filtered")
588 }
589 }
590 }
591 let errs = &["usr/foo/../../bar"];
592 for &k in errs {
593 assert!(normalize_validate_path(k.into(), allow_nonusr).is_err());
594 assert!(normalize_validate_path(k.into(), imp_default).is_err());
595 }
596 assert!(matches!(
597 normalize_validate_path("var/lib/foo".into(), composefs_and_new_ostree).unwrap(),
598 NormalizedPathResult::Normal(_)
599 ));
600 }
601
602 #[tokio::test]
603 async fn tar_filter() -> Result<()> {
604 let tempd = tempfile::tempdir()?;
605 let rootfs = &tempd.path().join("rootfs");
606
607 std::fs::create_dir_all(rootfs.join("etc/systemd/system"))?;
608 std::fs::write(rootfs.join("etc/systemd/system/foo.service"), "fooservice")?;
609 std::fs::write(rootfs.join("blah"), "blah")?;
610 let rootfs_tar_path = &tempd.path().join("rootfs.tar");
611 let rootfs_tar = std::fs::File::create(rootfs_tar_path)?;
612 let mut rootfs_tar = tar::Builder::new(rootfs_tar);
613 rootfs_tar.append_dir_all(".", rootfs)?;
614 let _ = rootfs_tar.into_inner()?;
615 let mut dest = Vec::new();
616 let src = tokio::io::BufReader::new(tokio::fs::File::open(rootfs_tar_path).await?);
617 let cap_tmpdir = Dir::open_ambient_dir(&tempd, cap_std::ambient_authority())?;
618 filter_tar_async(
619 src,
620 oci_image::MediaType::ImageLayer,
621 &mut dest,
622 &Default::default(),
623 cap_tmpdir,
624 )
625 .await?;
626 let dest = dest.as_slice();
627 let mut final_tar = tar::Archive::new(Cursor::new(dest));
628 let destdir = &tempd.path().join("destdir");
629 final_tar.unpack(destdir)?;
630 assert!(destdir.join("usr/etc/systemd/system/foo.service").exists());
631 assert!(!destdir.join("blah").exists());
632 Ok(())
633 }
634
635 #[tokio::test]
639 async fn tar_filter_pax_etc_remap() -> Result<()> {
640 let tempd = tempfile::tempdir()?;
641 let src_tar_path = tempd.path().join("src.tar");
642 let pax_path = "etc/ssl/certs/Főtanúsítvány.pem";
643
644 {
647 let mut builder = tar::Builder::new(std::fs::File::create(&src_tar_path)?);
648 let data = b"cert";
649 let mut header = tar::Header::new_gnu();
650 header.set_size(data.len() as u64);
651 header.set_mode(0o644);
652 header.set_entry_type(tar::EntryType::Regular);
653 header.set_cksum();
654 builder.append_pax_extensions([("path", pax_path.as_bytes())].into_iter())?;
655 builder.append_data(&mut header, pax_path, &data[..])?;
656 builder.into_inner()?;
657 }
658
659 let mut dest = Vec::new();
660 let src = tokio::io::BufReader::new(tokio::fs::File::open(&src_tar_path).await?);
661 let cap_tmpdir = Dir::open_ambient_dir(&tempd, cap_std::ambient_authority())?;
662 filter_tar_async(
663 src,
664 oci_image::MediaType::ImageLayer,
665 &mut dest,
666 &Default::default(),
667 cap_tmpdir,
668 )
669 .await?;
670
671 let mut found_remapped = false;
675 let mut archive = tar::Archive::new(Cursor::new(dest.as_slice()));
676 for entry in archive.entries()? {
677 let mut entry = entry?;
678 let entry_path = entry.path()?;
679 let entry_path = entry_path.to_string_lossy();
680 let entry_path = entry_path.trim_start_matches("./");
681 if entry_path == format!("usr/{pax_path}") {
682 found_remapped = true;
683 }
684 if let Some(pax) = entry.pax_extensions()? {
685 for ext_res in pax {
686 let ext = ext_res?;
687 if let Ok("path" | "linkpath") = ext.key() {
688 let value = String::from_utf8_lossy(ext.value_bytes());
689 let clean = value.trim_start_matches("./").trim_end_matches('\0');
690 assert!(
691 !clean.starts_with("etc/") && clean != "etc",
692 "PAX header still contains unremapped /etc path: {value}"
693 );
694 }
695 }
696 }
697 }
698 assert!(
699 found_remapped,
700 "Expected remapped file at usr/{pax_path} not found in output"
701 );
702 Ok(())
703 }
704}