Merge pull request #28 from BotCoder254/feat/log-chunk-sections

Use Docker archive API for source/artifacts, anonymous volumes for /w…

Author: BotCoder254

CLAUDE.md

@@ -84,11 +84,16 @@ source. Runners (`runner/` crate) hold one outbound WS (`/runner/ws`, Bearer tok
 SHA-256-hashed registration token shown once; heartbeat cadence comes from `hello_ack`),
 connect to Docker via `connect_with_defaults` (local socket/pipe OR remote daemon via
 `DOCKER_HOST` + `DOCKER_TLS_VERIFY` + `DOCKER_CERT_PATH`, rustls), execute each job in a
-hardened container via bollard (temp workspace bind-mounted at /workspace, tarball checkout
-with traversal-safe extraction that also drops symlink/hardlink entries, one keep-alive
+hardened container via bollard (daemon-managed anonymous volume at /workspace — NOT a host
+bind mount, so a runner that itself runs in a container sharing the host docker.sock still
+gets a populated /workspace; source is streamed IN via the Docker archive API/`docker cp`
+after the container starts, tarball repackaged into a traversal-safe tar that strips
+GitHub's top-level dir and drops symlink/hardlink entries; artifacts are streamed back OUT
+of `.overup/artifacts/` via the same archive API, one keep-alive
 container with `no-new-privileges` always plus cap-drop ALL by default, memory/CPU/pids
 limits, bridge|none|isolated per-job network, optional non-root user + read-only rootfs,
-steps as `exec` with `<shell> -c`, SIGKILL on cancel/timeout, 5 s Docker stats sampling for
+steps as `exec` with `<shell> -c`, SIGKILL on cancel/timeout, container removed with its
+anonymous volume, 5 s Docker stats sampling for
 CPU/memory/net/blkio metrics reported with `job_result`), and stream seq-numbered log
 chunks. `services/log_hub.rs` is the single log write path: mask FIRST (checkout tokens +
 confidential-looking env values; a per-job carry buffer catches secrets split across chunk
@@ -403,7 +408,8 @@ upgrades), `dashmap` (RunnerHub connection registry + LogHub broadcast/mask maps
 endpoint, region `auto`, presigned URLs; isolated in `services/r2.rs`), and the local
 `protocol` crate (shared WS message types + HMAC helpers). The `runner/` crate adds
 `bollard` 0.21 (Docker Engine API: image pull, container lifecycle, exec streams),
-`tokio-tungstenite` (rustls), `tar` + `flate2` (traversal-safe tarball extraction),
+`tokio-tungstenite` (rustls), `tar` + `flate2` + `bytes` (repackaging the source tarball
+into a traversal-safe tar streamed into the container via the Docker archive API),
 `tempfile` (per-job workspaces). Deliberately NOT used: `octocrab` (the reqwest helper in
 `services/github_app.rs` suffices), `git2` (tarball checkout via the API instead of cloning
 — heavy native dep on Windows).
@@ -500,9 +506,13 @@ endpoint, region `auto`, presigned URLs; isolated in `services/r2.rs`), and the
 - List filters are validated before SQL: status/conclusion/trigger allow-lists, length caps
   on branch/search/timestamps, and free-text search runs through a server-built ILIKE
   pattern with `\`, `%`, `_` escaped — user text only ever matches literally
-- Execution isolation (reference runner): per-job temp workspaces (deleted after), tarball
-  extraction drops non-`Normal` path components AND symlink/hardlink entries (traversal
-  defense — a link target can't be validated by path checks), 1 GiB tarball cap, jobs run
+- Execution isolation (reference runner): /workspace is a daemon-managed anonymous volume
+  (removed with the container), NOT a host bind — source is repackaged into a
+  traversal-safe tar (top-level dir stripped, non-`Normal` path components AND
+  symlink/hardlink entries dropped — a link target can't be validated by path checks) and
+  streamed IN over the Docker archive API after the container starts (so a containerized
+  runner sharing the host docker.sock still sees a populated /workspace); artifacts are
+  streamed back OUT of `.overup/artifacts/` the same way, 1 GiB tarball cap, jobs run
   in Docker containers with `no-new-privileges` always set, **capabilities dropped by
   default** (`RUNNER_CAP_DROP=false` to opt out), memory/CPU/pids limits
   (`RUNNER_JOB_MEMORY_BYTES`/`RUNNER_JOB_NANO_CPUS`/`RUNNER_JOB_PIDS_LIMIT`, 0 disables),

backend/.env.example

@@ -62,11 +62,15 @@ RUNNER_JOB_SIGNING_KEY=
 
 # Container image for jobs that declare no container and no known runs-on
 # label. The catthehacker/ubuntu:act-* family (what nektos/act and Gitea
-# Actions default to) is GitHub-runner-compatible with Node/npm/yarn,
-# Python, git, build-essential etc. preinstalled (~500 MB compressed). For
-# a specific toolchain, jobs pin their own image in workflow YAML with the
-# GitHub `container:` key, e.g. `container: rust:1`, `container: node:22`,
-# `container: golang:1.23`.
+# Actions default to) is the MEDIUM GitHub-runner-compatible image with
+# Node/npm/yarn, Python, git, build-essential etc. preinstalled
+# (~500 MB compressed). It does NOT ship every toolchain — notably no Rust
+# (`cargo: not found`) or Go — so a job needing one pins its own image in
+# workflow YAML with the GitHub `container:` key, e.g. `container: rust:1`,
+# `container: node:22`, `container: golang:1.23`. (`uses: actions/setup-*`
+# steps are not executed; overup emits a plan notice pointing at the
+# equivalent `container:` pin. `uses: actions/checkout` is a no-op — the
+# runner checks the repository out into /workspace automatically.)
 DEFAULT_JOB_IMAGE=catthehacker/ubuntu:act-latest
 
 # --- Secrets Management (optional but recommended) ---------------------------

backend/src/services/pipeline_plan.rs

@@ -105,12 +105,20 @@ pub fn build_plans(raw_content: &str, default_image: &str) -> Result<Vec<Planned
             for (index, step) in list.iter().enumerate() {
                 if let Some(uses) = step.get("uses").and_then(Value::as_str) {
                     let uses = sanitize_for_notice(uses);
-                    match setup_action_hint(&uses) {
-                        Some(hint) => notices.push(format!(
+                    if is_checkout_action(&uses) {
+                        // Checkout is NOT skipped — the runner fetches the
+                        // repository automatically before the job runs. Saying
+                        // "not supported; skipped" wrongly implies no source.
+                        notices.push(format!(
+                            "step `uses: {uses}` is a no-op here — your repository is checked out automatically before the job runs"
+                        ));
+                    } else if let Some(hint) = setup_action_hint(&uses) {
+                        notices.push(format!(
                             "step `uses: {uses}` is not executed; the default job image ships common toolchains — pin this one by adding `{hint}` to the job"
-                        )),
-                        None => notices
-                            .push(format!("step `uses: {uses}` is not supported yet; skipped")),
+                        ));
+                    } else {
+                        notices
+                            .push(format!("step `uses: {uses}` is not supported yet; skipped"));
                     }
                     continue;
                 }
@@ -243,6 +251,14 @@ fn sanitize_for_notice(value: &str) -> String {
     value.chars().filter(|c| !c.is_control()).take(100).collect()
 }
 
+/// Whether a `uses:` step is a repository checkout. These are a no-op on
+/// overup because the runner checks the repository out automatically before
+/// the job runs — so the notice must not imply the source is missing.
+fn is_checkout_action(uses: &str) -> bool {
+    let action = uses.split('@').next().unwrap_or(uses);
+    matches!(action, "actions/checkout")
+}
+
 /// Well-known `setup-*` actions mapped to the per-job `container:` image
 /// that provides the same toolchain. `uses:` steps are not executed on
 /// overup, so the notice points authors at the mechanism that is.
@@ -466,4 +482,22 @@ jobs:
         assert_eq!(setup_action_hint("dtolnay/rust-toolchain@stable"), Some("container: rust:1"));
         assert_eq!(setup_action_hint("actions/checkout@v4"), None);
     }
+
+    #[test]
+    fn checkout_notice_is_reassuring_not_a_skip_warning() {
+        // The SIMPLE workflow uses `actions/checkout@v4`; its notice must say
+        // the repo is checked out automatically, NOT "not supported; skipped".
+        let plans = build_plans(SIMPLE, "ubuntu:24.04").ok().unwrap();
+        let notices = plans[0].plan["notices"].as_array().unwrap();
+        let checkout = notices
+            .iter()
+            .filter_map(|n| n.as_str())
+            .find(|n| n.contains("actions/checkout@v4"))
+            .expect("a checkout notice");
+        assert!(checkout.contains("checked out automatically"), "{checkout}");
+        assert!(!checkout.contains("not supported"), "{checkout}");
+
+        assert!(is_checkout_action("actions/checkout@v4"));
+        assert!(!is_checkout_action("actions/setup-node@v4"));
+    }
 }

runner/Cargo.lock

@@ -8,6 +8,9 @@ anyhow = "1"
 # `ssl` (rustls + ring) enables remote TLS daemons: DOCKER_HOST=tcp://…:2376
 # with DOCKER_TLS_VERIFY=1 and DOCKER_CERT_PATH pointing at ca/cert/key.pem.
 bollard = { version = "0.21", features = ["ssl"] }
+# Source/artifacts move over the Docker archive API (docker cp), so we hand
+# bollard a `Bytes` tar body via `bollard::body_full`.
+bytes = "1"
 chrono = { version = "0.4", features = ["serde"] }
 dotenvy = "0.15"
 flate2 = "1"