Skip to content

Commit 6e5b475

Browse files
committed
feat(fs/mem): tarfs implementation
This implements the internal Hermit image mode / Initramfs support. The decompressed Hermit image gets passed from the bootloader (hermit-loader) or hypervisor (uhyve) to the kernel, the location is announced via the FDT property `/chosen/linux,initrd-*`, and the kernel parses the contained tar (expected in ustar format) file, placing its content at the filesystem root, similar to an initrd. The access permissions get copied from the tar archive, directories get default permissions. - feat(xtask): initramfs building
1 parent 76e9fc6 commit 6e5b475

8 files changed

Lines changed: 487 additions & 21 deletions

File tree

Cargo.lock

Lines changed: 289 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,13 @@ semihosting = ["dep:semihosting"]
164164
## _application processor_s (AP) from the _boot-strap processor_ (BSP).
165165
smp = ["acpi"]
166166

167+
## Enables (internal) hermit image / initramfs support.
168+
##
169+
## This allows packaging all the kernel's static environment files into a .tar.gz archive,
170+
## whose decompressed version gets handled by the kernel
171+
## (the decompression gets handled by the bootloader / hypervisor, for now).
172+
initramfs = ["dep:tar-no-std"]
173+
167174
#! ### Network Features
168175

169176
## Enables TCP support.
@@ -370,6 +377,7 @@ simple-shell = { version = "0.0.1", optional = true }
370377
smallvec = { version = "1", features = ["const_new"] }
371378
take-static = "0.1"
372379
talc = { version = "5" }
380+
tar-no-std = { version = "0.4", optional = true, features = ["alloc"] }
373381
thiserror = { version = "2", default-features = false }
374382
time = { version = "0.3", default-features = false }
375383
uhyve-interface = { version = "0.3", optional = true }

src/fs/mem.rs

Lines changed: 74 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -122,15 +122,6 @@ pub(crate) struct RamFileInner {
122122
pub attr: FileAttr,
123123
}
124124

125-
impl RamFileInner {
126-
pub fn new(attr: FileAttr) -> Self {
127-
Self {
128-
data: Vec::new(),
129-
attr,
130-
}
131-
}
132-
}
133-
134125
pub struct RamFileInterface {
135126
/// Position within the file
136127
pos: Mutex<usize>,
@@ -354,6 +345,10 @@ impl VfsNode for RamFile {
354345

355346
impl RamFile {
356347
pub fn new(mode: AccessPermission) -> Self {
348+
Self::new_with_data(Vec::new(), mode)
349+
}
350+
351+
fn new_with_data(data: Vec<u8>, mode: AccessPermission) -> Self {
357352
let microseconds = arch::kernel::systemtime::now_micros();
358353
let t = timespec::from_usec(microseconds as i64);
359354
let attr = FileAttr {
@@ -365,7 +360,7 @@ impl RamFile {
365360
};
366361

367362
Self {
368-
data: Arc::new(RwLock::new(RamFileInner::new(attr))),
363+
data: Arc::new(RwLock::new(RamFileInner { data, attr })),
369364
}
370365
}
371366
}
@@ -468,6 +463,65 @@ impl MemDirectory {
468463
}
469464
}
470465

466+
#[cfg(feature = "initramfs")]
467+
pub fn try_extend_from_image(&self, image: &'static [u8]) -> io::Result<()> {
468+
let tar_archive_ref = tar_no_std::TarArchiveRef::new(image).map_err(|e| {
469+
error!("[Hermit image] Tar file has invalid format: {e:?}");
470+
Errno::Inval
471+
})?;
472+
473+
for entry in tar_archive_ref.entries() {
474+
let filename = entry.filename();
475+
let filename = filename.as_str().map_err(|e| {
476+
error!(
477+
"[Hermit image] Tar entry has not supported filename (non UTF-8): {filename:?}; {e}",
478+
);
479+
Errno::Inval
480+
})?;
481+
if filename.is_empty() {
482+
continue;
483+
}
484+
debug!("[Hermit image] Processing tar entry: {filename}");
485+
486+
let mode = match entry.posix_header().mode.to_flags() {
487+
Ok(mode) => mode,
488+
Err(e) => {
489+
error!(
490+
"[Hermit image] Tar entry {filename:?} has invalid mode: {:?}; {e}",
491+
entry.posix_header().mode,
492+
);
493+
continue;
494+
}
495+
};
496+
let mode = AccessPermission::from_bits(mode.bits() as u32).ok_or_else(|| {
497+
error!("[Hermit image] Tar entry {filename:?} has invalid mode: {mode:?}");
498+
Errno::Inval
499+
})?;
500+
501+
for (i, _) in filename.match_indices("/") {
502+
let part = &filename[..i];
503+
if self.traverse_lstat(part).is_err() {
504+
self.traverse_mkdir(
505+
part,
506+
AccessPermission::S_IRUSR
507+
| AccessPermission::S_IWUSR
508+
| AccessPermission::S_IRGRP,
509+
)
510+
.inspect_err(|e| {
511+
error!("[Hermit image] Unable to mkdir {part:?}: {e}");
512+
})?;
513+
}
514+
}
515+
516+
self.traverse_create_file(filename, entry.data(), mode)
517+
.inspect_err(|e| {
518+
error!("[Hermit image] Unable to write entry {filename:?}: {e}");
519+
})?;
520+
}
521+
522+
Ok(())
523+
}
524+
471525
async fn async_traverse_open(
472526
&self,
473527
path: &str,
@@ -701,11 +755,16 @@ impl VfsNode for MemDirectory {
701755
return directory.traverse_create_file(rest, data, mode);
702756
}
703757

704-
let file = RomFile::new(data, mode);
705-
self.inner
706-
.write()
707-
.await
708-
.insert(component.to_owned(), Box::new(file));
758+
let file: Box<dyn VfsNode> = if mode.contains(AccessPermission::S_IWUSR)
759+
|| mode.contains(AccessPermission::S_IWGRP)
760+
|| mode.contains(AccessPermission::S_IWOTH)
761+
{
762+
Box::new(RamFile::new_with_data(data.to_vec(), mode))
763+
} else {
764+
Box::new(RomFile::new(data, mode))
765+
};
766+
767+
self.inner.write().await.insert(component.to_owned(), file);
709768
Ok(())
710769
},
711770
None,

src/fs/mod.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use hermit_sync::{InterruptSpinMutex, OnceCell};
1919
use mem::MemDirectory;
2020
use num_enum::{IntoPrimitive, TryFromPrimitive};
2121

22+
use crate::env::{StartInfo as _, start_info};
2223
use crate::errno::Errno;
2324
use crate::executor::block_on;
2425
use crate::fd::{AccessPermission, Fd, ObjectInterface, OpenOption, insert_object, remove_object};
@@ -309,6 +310,22 @@ pub(crate) fn init() {
309310

310311
let root_filesystem = Filesystem::new();
311312

313+
// Handle optional Initramfs specified in start info modules.
314+
#[cfg(feature = "initramfs")]
315+
for i in start_info().modules() {
316+
root_filesystem
317+
.root
318+
.try_extend_from_image(i)
319+
.expect("Unable to parse initramfs");
320+
}
321+
322+
#[cfg(not(feature = "initramfs"))]
323+
if start_info().modules().next().is_some() {
324+
error!(
325+
"Kernel built without Hermit image support, but a Hermit image was supplied: ignoring"
326+
);
327+
}
328+
312329
root_filesystem
313330
.mkdir("/tmp", AccessPermission::from_bits(0o777).unwrap())
314331
.expect("Unable to create /tmp");

src/mm/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ pub(crate) fn claim_initial_heap() {
9191

9292
#[cfg(target_os = "none")]
9393
pub(crate) fn init() {
94-
use crate::arch::mm::paging;
94+
use arch::mm::paging;
9595

9696
unsafe {
9797
arch::mm::init();
@@ -130,7 +130,7 @@ pub(crate) fn init() {
130130
// we reserve at least 75% of the memory for the user space
131131
let reserve: usize = (avail_mem * 75) / 100;
132132
// 64 MB is enough as kernel heap
133-
let reserve = core::cmp::min(reserve, 0x0400_0000);
133+
let reserve = cmp::min(reserve, 0x0400_0000);
134134

135135
let virt_size: usize = reserve.align_down(LargePageSize::SIZE as usize);
136136
let layout = PageLayout::from_size_align(virt_size, LargePageSize::SIZE as usize).unwrap();

xtask/Cargo.toml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,39 @@ edition = "2024"
66
[features]
77
default = ["ci"]
88
ci = [
9+
"dep:cargo_metadata",
10+
"dep:flate2",
11+
"dep:hermit-entry",
912
"dep:libc",
1013
"dep:ovmf-prebuilt",
1114
"dep:shlex",
1215
"dep:sysinfo",
16+
"dep:tar",
17+
"dep:toml",
1318
"dep:ureq",
1419
"dep:vsock",
1520
"dep:wait-timeout",
21+
"dep:walkdir",
1622
]
1723

1824
[dependencies]
1925
anyhow = "1.0"
26+
cargo_metadata = { version = "0.23", optional = true }
2027
clap = { version = "4", features = ["derive"] }
28+
flate2 = { version = "1.1", optional = true }
2129
goblin = { version = "0.10", default-features = false, features = ["archive", "elf32", "elf64", "std"] }
30+
hermit-entry = { version = "0.10", features = ["loader"], optional = true }
2231
home = "0.5"
2332
libc = { version = "0.2", optional = true }
2433
ovmf-prebuilt = { version = "0.2", optional = true }
2534
shlex = { version = "2", optional = true }
2635
sysinfo = { version = "0.39", optional = true }
36+
tar = { version = "0.4", optional = true }
37+
toml = { version = "1.1", optional = true }
2738
ureq = { version = "3", default-features = false, features = ["rustls"], optional = true }
2839
vsock = { version = "0.5", optional = true }
2940
wait-timeout = { version = "0.2", optional = true }
41+
walkdir = { version = "2.5", optional = true }
3042
xshell = "0.2"
3143

3244
[lints]

xtask/src/ci/rs.rs

Lines changed: 84 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
use std::path::PathBuf;
1+
use std::fs;
2+
use std::path::{Path, PathBuf};
23

34
use anyhow::Result;
45
use clap::{Args, Subcommand};
6+
use hermit_entry::config;
57

68
use crate::cargo_build::CargoBuild;
79

@@ -58,6 +60,7 @@ impl Rs {
5860
}
5961

6062
let mut cargo = crate::cargo();
63+
let parent_root = super::parent_root();
6164

6265
if self.package.contains("rftrace") {
6366
cargo.env(
@@ -67,7 +70,7 @@ impl Rs {
6770
};
6871

6972
cargo
70-
.current_dir(super::parent_root())
73+
.current_dir(parent_root)
7174
.arg("build")
7275
.args(self.cargo_build.artifact.arch.ci_cargo_args())
7376
.args(self.cargo_build.cargo_build_args())
@@ -77,10 +80,88 @@ impl Rs {
7780
let status = cargo.status()?;
7881
assert!(status.success());
7982

83+
// discover possible initramfs seed
84+
let manifest_dir = {
85+
let cur_package = cargo_metadata::PackageName::new(self.package.clone());
86+
let mut cargo = cargo_metadata::MetadataCommand::new();
87+
cargo
88+
.current_dir(parent_root)
89+
.no_deps()
90+
.verbose(true)
91+
.exec()?
92+
.packages
93+
.iter()
94+
.find(|i| i.name == cur_package)
95+
.expect("unable to find current package in `cargo metadata` output")
96+
// this path points to `Cargo.toml`
97+
.manifest_path
98+
.parent()
99+
.unwrap()
100+
.to_path_buf()
101+
};
102+
eprintln!("MANIFEST_DIR = {manifest_dir}");
103+
let maybe_initramfs = {
104+
let initramfs = manifest_dir.join("initramfs");
105+
if initramfs.is_dir() {
106+
Some(initramfs)
107+
} else {
108+
None
109+
}
110+
};
111+
112+
let mut build_artifact = self.cargo_build.artifact.ci_image(&self.package);
113+
114+
// handle possible initramfs
115+
if let Some(initramfs) = maybe_initramfs {
116+
eprintln!("discovered initramfs seed, creating initramfs.");
117+
// find kernel name
118+
let konfig = fs::read_to_string(initramfs.join(config::Config::DEFAULT_PATH))?;
119+
let konfig: config::Config<'_> = toml::from_str(&konfig)?;
120+
let kernel_name: &str = match &konfig {
121+
config::Config::V1 { kernel, .. } => kernel,
122+
};
123+
124+
let tar_artifact_path = build_artifact.with_extension("tar.gz");
125+
let mut tar_artifact = tar::Builder::new(flate2::write::GzEncoder::new(
126+
fs::File::create(&tar_artifact_path)?,
127+
flate2::Compression::default(),
128+
));
129+
tar_artifact.mode(tar::HeaderMode::Deterministic);
130+
131+
// NOTE: use tar ustar to create the image.
132+
133+
// add kernel
134+
eprintln!("- {kernel_name}");
135+
{
136+
let mut header = tar::Header::new_ustar();
137+
let kernel_meta = fs::metadata(&build_artifact)?;
138+
header.set_path(kernel_name).unwrap();
139+
header.set_size(kernel_meta.len());
140+
header.set_cksum();
141+
142+
tar_artifact.append(&header, fs::File::open(&build_artifact)?)?;
143+
}
144+
145+
// add rest
146+
for entry in walkdir::WalkDir::new(&initramfs) {
147+
let entry = entry?;
148+
let entry_rel_path = entry.path().strip_prefix(&initramfs)?;
149+
eprintln!("- {}", entry_rel_path.display());
150+
if entry_rel_path == Path::new(kernel_name) || entry.metadata()?.is_dir() {
151+
continue;
152+
}
153+
154+
tar_artifact.append_path_with_name(entry.path(), entry_rel_path)?;
155+
}
156+
157+
tar_artifact.into_inner()?.finish()?.sync_all()?;
158+
build_artifact = tar_artifact_path;
159+
}
160+
80161
if super::in_ci() {
81162
eprintln!("::endgroup::");
82163
}
83164

84-
Ok(self.cargo_build.artifact.ci_image(&self.package))
165+
Ok(build_artifact)
85166
}
86167
}

xtask/src/clippy.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ impl Clippy {
2121
clippy().run()?;
2222
clippy().arg("--features=common-os").run()?;
2323
clippy()
24-
.arg("--features=acpi,dns,fsgsbase,pci,smp,vga")
24+
.arg("--features=acpi,dns,fsgsbase,pci,smp,vga,initramfs")
2525
.run()?;
2626
clippy().arg("--no-default-features").run()?;
2727
clippy().arg("--all-features").run()?;

0 commit comments

Comments
 (0)