feat(orion-server): queue builds when no idle worker for up to 60m (#2163)

* feat(orion-server): queue builds when no idle worker for up to 60m

Restore FIFO enqueue when VMs are offline or busy instead of immediately
marking builds Interrupted, and re-queue on dispatch races.

* feat(cl): show Orion build status on CL list

Return aggregated latest-task build_status from the list API and render
it with the shared Checks status icons.

Author: benjamin-747

Cargo.lock

@@ -111,7 +111,7 @@ dashmap = "6.2.1"
 once_cell = "1.21.4"
 serial_test = "4.0.1"
 sysinfo = "0.39.6"
-http = "1.4.2"
+http = "1.5.0"
 url = "2.5.8"
 jemallocator = "0.5.4"
 mimalloc = "0.1.52"
@@ -123,9 +123,9 @@ bs58 = "0.5.1"
 indexmap = "2.14"
 envsubst = "0.2.1"
 directories = "6.0.0"
-redis = "1.4.1"
+redis = "1.5.0"
 redis-test = "1.0.4"
-rustls = "0.23.42"
+rustls = "0.23.43"
 object_store = "0.14.1"
 parse-display = "0.11.0"
 toml = "1.1.3"

Cargo.toml

@@ -24,6 +24,21 @@ impl ClApplicationService {
             .bot_names_among(&names)
             .await?;
         ItemRes::apply_bot_flags(&mut items, &bot_names);
+
+        let links: Vec<String> = items.iter().map(|i| i.link.clone()).collect();
+        match self
+            .storage()
+            .cl_service
+            .cl_store()
+            .latest_build_status_by_cl_links(&links)
+            .await
+        {
+            Ok(statuses) => ItemRes::apply_build_statuses(&mut items, &statuses),
+            Err(e) => {
+                tracing::warn!("Failed to load CL build statuses for list: {e}");
+            }
+        }
+
         Ok(CommonPage { items, total })
     }
 

ceres/src/application/api_service/mono/cl_list.rs

@@ -28,6 +28,10 @@ pub struct ItemRes {
     pub labels: Vec<LabelItem>,
     pub assignees: Vec<String>,
     pub comment_num: usize,
+    /// Aggregated Orion build status for this CL (latest task), if any.
+    /// Issue list leaves this unset/`null`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build_status: Option<String>,
 }
 
 impl From<ItemDetails> for ItemRes {
@@ -47,6 +51,7 @@ impl From<ItemDetails> for ItemRes {
                 labels: value.labels.into_iter().map(|m| m.into()).collect(),
                 assignees: value.assignees,
                 comment_num: value.comment_num,
+                build_status: None,
             },
             ItemKind::Cl(model) => Self {
                 id: model.id,
@@ -62,6 +67,7 @@ impl From<ItemDetails> for ItemRes {
                 labels: value.labels.into_iter().map(|m| m.into()).collect(),
                 assignees: value.assignees,
                 comment_num: value.comment_num,
+                build_status: None,
             },
         }
     }
@@ -73,6 +79,15 @@ impl ItemRes {
             item.author_is_bot = bot_names.contains(&item.author) || item.author == "system";
         }
     }
+
+    pub fn apply_build_statuses(
+        items: &mut [Self],
+        statuses: &std::collections::HashMap<String, String>,
+    ) {
+        for item in items {
+            item.build_status = statuses.get(&item.link).cloned();
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, ToSchema)]

ceres/src/model/issue.rs

@@ -447,4 +447,74 @@ impl ClStorage {
         self.batch_save_model(save_models).await?;
         Ok(())
     }
+
+    /// For each CL link, resolve the latest Orion task and aggregate its
+    /// `build_targets.latest_state` with Checks-compatible worst-wins priority:
+    /// Failed > Interrupted > Building > Pending/Uninitialized > Completed.
+    pub async fn latest_build_status_by_cl_links(
+        &self,
+        links: &[String],
+    ) -> Result<HashMap<String, String>, MegaError> {
+        if links.is_empty() {
+            return Ok(HashMap::new());
+        }
+
+        let tasks = callisto::orion_tasks::Entity::find()
+            .filter(callisto::orion_tasks::Column::Cl.is_in(links.to_vec()))
+            .order_by_desc(callisto::orion_tasks::Column::CreatedAt)
+            .all(self.get_connection())
+            .await?;
+
+        // First row per CL wins (already ordered by created_at DESC).
+        let mut latest_task_by_cl: HashMap<String, uuid::Uuid> = HashMap::new();
+        for task in tasks {
+            latest_task_by_cl.entry(task.cl).or_insert(task.id);
+        }
+
+        if latest_task_by_cl.is_empty() {
+            return Ok(HashMap::new());
+        }
+
+        let task_ids: Vec<uuid::Uuid> = latest_task_by_cl.values().copied().collect();
+        let targets = callisto::build_targets::Entity::find()
+            .filter(callisto::build_targets::Column::TaskId.is_in(task_ids))
+            .all(self.get_connection())
+            .await?;
+
+        let mut states_by_task: HashMap<uuid::Uuid, Vec<String>> = HashMap::new();
+        for target in targets {
+            states_by_task
+                .entry(target.task_id)
+                .or_default()
+                .push(target.latest_state);
+        }
+
+        let mut result = HashMap::new();
+        for (cl, task_id) in latest_task_by_cl {
+            if let Some(states) = states_by_task.get(&task_id)
+                && let Some(status) = aggregate_target_states(states)
+            {
+                result.insert(cl, status);
+            }
+        }
+        Ok(result)
+    }
+}
+
+/// Worst-wins aggregation matching Checks UI `getTaskStatus` priority.
+fn aggregate_target_states(states: &[String]) -> Option<String> {
+    if states.is_empty() {
+        return None;
+    }
+    let priority = |s: &str| -> u8 {
+        match s {
+            "Failed" => 0,
+            "Interrupted" => 1,
+            "Building" => 2,
+            "Pending" | "Uninitialized" => 3,
+            "Completed" => 4,
+            _ => 3,
+        }
+    };
+    states.iter().min_by_key(|s| priority(s.as_str())).cloned()
 }

jupiter/src/storage/cl_storage.rs

@@ -1,6 +1,6 @@
 # ────── Stage 0: Chef ──────
 # Downloads and installs cargo-chef (prebuilt binary, no Rust compilation needed).
-FROM rust:1.96.0 AS chef
+FROM rust:1.97.1 AS chef
 
 ARG CARGO_CHEF_VERSION=0.1.77
 

mono/Dockerfile

@@ -1,13 +1,16 @@
 import React, { forwardRef, memo, ReactNode } from 'react'
 
 import { PostApiIssueListData } from '@gitmono/types'
-import { Button, ChatBubbleIcon, Command, ConditionalWrap, LoadingSpinner, useCommand } from '@gitmono/ui'
+import { Button, ChatBubbleIcon, Command, ConditionalWrap, LoadingSpinner, Tooltip, useCommand } from '@gitmono/ui'
 
 import { MemberHoverAvatarList } from '@/components/Issues/MemberHoverAvatarList'
 import { SubjectCommand } from '@/components/Subject/SubjectCommand'
 import { BreadcrumbTitlebarContainer } from '@/components/Titlebar/BreadcrumbTitlebar'
 import { getFontColor } from '@/utils/getFontColor'
 
+import { identifyStatus } from './components/Checks/cpns/identifyStatus'
+import { Status } from './components/Checks/cpns/store'
+
 export function List<T>({
   lists,
   header,
@@ -196,9 +199,21 @@ export const ItemLabels = ({ item }: { item: ItemsType[number] }) => {
 }
 
 export const ItemRightIcons = ({ item }: { item: ItemsType[number] }) => {
+  const buildStatus = item.build_status
+  const statusEnum =
+    buildStatus && Object.values(Status).includes(buildStatus as Status) ? (buildStatus as Status) : null
+
   return (
     // <div className='mr-10 flex w-fit items-center justify-between gap-10'>
     <div className='flex items-center gap-4'>
+      {statusEnum ? (
+        <Tooltip label={statusEnum}>
+          <span className='flex items-center' aria-label={`Build ${statusEnum}`}>
+            {identifyStatus(statusEnum)}
+          </span>
+        </Tooltip>
+      ) : null}
+
       <div
         style={{
           visibility: `${item.comment_num === 0 ? 'hidden' : 'unset'}`

moon/apps/web/components/ClView/ClList.tsx

@@ -8,6 +8,7 @@ import { LoadingSpinner } from '@gitmono/ui/Spinner'
 import { usePostRetryBuild } from '@/hooks/SSE/usePostRetryBuild'
 
 import { TERMINAL_BUILD_STATUSES } from '../hooks/logUtils'
+import { identifyStatus } from './identifyStatus'
 import { BuildDTO, getLatestBuildId, isTaskQueued, Status, TaskInfoDTO } from './store'
 
 /**
@@ -371,24 +372,4 @@ const TaskItem = memo(function TaskItem({
   )
 })
 
-export const identifyStatus = (status: Status[keyof Status]) => {
-  switch (status) {
-    case Status.Completed:
-      return <CheckIcon size={14} className='text-green-700 dark:text-green-400' />
-    case Status.Failed:
-      return <XIcon size={14} className='text-red-600 dark:text-red-400' />
-    case Status.Interrupted:
-      return <XIcon size={14} className='text-orange-600 dark:text-orange-400' />
-    case Status.Building:
-      return <LoadingSpinner />
-    case Status.Pending:
-      return <LoadingSpinner />
-    case Status.Uninitialized:
-      return <LoadingSpinner />
-    case Status.NotFound:
-      return <LoadingSpinner />
-
-    default:
-      return <XIcon size={14} className='text-red-600 dark:text-red-400' />
-  }
-}
+export { identifyStatus } from './identifyStatus'

moon/apps/web/components/ClView/components/Checks/cpns/Task.tsx

@@ -0,0 +1,27 @@
+import { CheckIcon, XIcon } from '@primer/octicons-react'
+
+import { LoadingSpinner } from '@gitmono/ui/Spinner'
+
+import { Status } from './store'
+
+export const identifyStatus = (status: Status[keyof Status]) => {
+  switch (status) {
+    case Status.Completed:
+      return <CheckIcon size={14} className='text-green-700 dark:text-green-400' />
+    case Status.Failed:
+      return <XIcon size={14} className='text-red-600 dark:text-red-400' />
+    case Status.Interrupted:
+      return <XIcon size={14} className='text-orange-600 dark:text-orange-400' />
+    case Status.Building:
+      return <LoadingSpinner />
+    case Status.Pending:
+      return <LoadingSpinner />
+    case Status.Uninitialized:
+      return <LoadingSpinner />
+    case Status.NotFound:
+      return <LoadingSpinner />
+
+    default:
+      return <XIcon size={14} className='text-red-600 dark:text-red-400' />
+  }
+}