Merge pull request #32 from BotCoder254/feat/log-chunk-sections

fix(sync): re-parse stored workflows after parser upgrades so detection lights up

Author: BotCoder254

backend/src/db/workflows.rs

@@ -6,18 +6,32 @@ use uuid::Uuid;
 use crate::models::workflow::{WorkflowDetailRow, WorkflowJobRow, WorkflowSummaryRow};
 use crate::services::workflow_parse::ParsedJob;
 
-/// blob shas of the workflows currently stored for a repository — sync
-/// compares against GitHub's listing to fetch only changed files.
-pub async fn blob_shas_for_repo(
+/// Per-path sync state of the workflows currently stored for a repository:
+/// blob sha (fetch only changed files) and the stamped parser version
+/// (re-parse stored files after a parser upgrade even when the sha is
+/// unchanged — otherwise new metadata never materializes). The `jsonb_typeof`
+/// guard reads legacy/hand-edited metadata as version 0, never an error.
+pub async fn sync_states_for_repo(
     pool: &PgPool,
     repository_id: Uuid,
-) -> sqlx::Result<HashMap<String, String>> {
-    let rows: Vec<(String, String)> =
-        sqlx::query_as("SELECT path, blob_sha FROM workflows WHERE repository_id = $1")
-            .bind(repository_id)
-            .fetch_all(pool)
-            .await?;
-    Ok(rows.into_iter().collect())
+) -> sqlx::Result<HashMap<String, (String, i64)>> {
+    let rows: Vec<(String, String, i64)> = sqlx::query_as(
+        r#"
+        SELECT path, blob_sha,
+               CASE WHEN jsonb_typeof(metadata->'parserVersion') = 'number'
+                    THEN floor((metadata->>'parserVersion')::numeric)::bigint
+                    ELSE 0 END
+        FROM workflows
+        WHERE repository_id = $1
+        "#,
+    )
+    .bind(repository_id)
+    .fetch_all(pool)
+    .await?;
+    Ok(rows
+        .into_iter()
+        .map(|(path, sha, version)| (path, (sha, version)))
+        .collect())
 }
 
 /// Upsert one workflow and replace its job rows. Runs inside the sync

backend/src/services/repo_sync.rs

@@ -135,16 +135,22 @@ async fn run_sync(
         .take(MAX_WORKFLOW_FILES)
         .collect();
 
-    let known = db::workflows::blob_shas_for_repo(&state.pool, repository.id)
+    let known = db::workflows::sync_states_for_repo(&state.pool, repository.id)
         .await
         .map_err(|e| fail("database error")(e.into()))?;
 
     // Fetch + parse changed files outside the transaction; persist inside it.
     let mut parsed_files = Vec::new();
     let mut fetched = 0usize;
     for entry in &eligible {
-        if known.get(&entry.path).map(String::as_str) == Some(entry.sha.as_str()) {
-            continue; // unchanged
+        // Skip only when the content is unchanged AND it was parsed by the
+        // current parser — a parser upgrade re-parses stored files once so
+        // new metadata (secretRefs/varRefs/environments) materializes for
+        // workflows that never change on GitHub.
+        if known.get(&entry.path).is_some_and(|(sha, version)| {
+            sha == &entry.sha && *version == workflow_parse::PARSER_VERSION
+        }) {
+            continue;
         }
         if entry.size as usize > github_app::MAX_WORKFLOW_FILE_BYTES {
             // Oversized files become an errored catalog entry, not a fetch.
@@ -314,7 +320,9 @@ fn oversize_placeholder() -> workflow_parse::ParsedWorkflow {
         name: None,
         triggers: Vec::new(),
         jobs: Vec::new(),
-        metadata: serde_json::json!({}),
+        // Version-stamped so sync doesn't re-process the oversized file on
+        // every run.
+        metadata: serde_json::json!({ "parserVersion": workflow_parse::PARSER_VERSION }),
         diagnostics: vec![workflow_parse::Diagnostic {
             severity: workflow_parse::Severity::Error,
             message: "workflow file exceeds the size limit and was not parsed".into(),

backend/src/services/workflow_parse.rs

@@ -13,6 +13,13 @@ use serde_yaml_ng::{Mapping, Value};
 
 use super::github_app::MAX_WORKFLOW_FILE_BYTES;
 
+/// Stamped into every workflow's `metadata` JSONB. Bump whenever the parse
+/// output or metadata shape changes: repo sync re-parses any stored workflow
+/// whose stamped version is older, even when its blob sha is unchanged —
+/// otherwise new metadata (e.g. the detected-requirements ref arrays) would
+/// never materialize for files that don't change on GitHub.
+pub const PARSER_VERSION: i64 = 2;
+
 /// Resource budgets: a workflow file that exceeds these is hostile or
 /// broken, not "large". They bound both memory and walk time.
 const MAX_NODES: usize = 20_000;
@@ -141,7 +148,9 @@ impl ParsedWorkflow {
             name: None,
             triggers: Vec::new(),
             jobs: Vec::new(),
-            metadata: serde_json::json!({}),
+            // Version-stamped even on failure so sync doesn't re-parse a
+            // persistently broken file on every run.
+            metadata: serde_json::json!({ "parserVersion": PARSER_VERSION }),
             diagnostics,
         }
     }
@@ -206,6 +215,7 @@ pub fn parse_and_validate(content: &str) -> ParsedWorkflow {
     validate_needs(&jobs, &mut diagnostics);
 
     let metadata = serde_json::json!({
+        "parserVersion": PARSER_VERSION,
         "permissions": get(root, "permissions").map(value_to_json),
         "concurrency": get(root, "concurrency").map(value_to_json),
         "envKeys": get(root, "env")
@@ -1106,4 +1116,20 @@ jobs:
         assert_eq!(parsed.metadata["varRefs"], serde_json::json!([]));
         assert_eq!(parsed.metadata["environments"], serde_json::json!([]));
     }
+
+    #[test]
+    fn metadata_is_parser_version_stamped_on_success_and_failure() {
+        let parsed = parse_and_validate(
+            "on: push\njobs:\n  a:\n    runs-on: ubuntu-latest\n    steps: []\n",
+        );
+        assert_eq!(
+            parsed.metadata["parserVersion"],
+            serde_json::json!(PARSER_VERSION)
+        );
+        let broken = parse_and_validate("{not yaml");
+        assert_eq!(
+            broken.metadata["parserVersion"],
+            serde_json::json!(PARSER_VERSION)
+        );
+    }
 }

src/features/environments/pages/EnvironmentsPage.tsx

@@ -164,7 +164,7 @@ export function EnvironmentsPage() {
                 description={
                   q
                     ? 'Nothing matches the current search. Clear it to see every environment.'
-                    : 'Create environments like production or staging, give each its own secrets, and reference them from workflow YAML with `environment: <name>`.'
+                    : 'Create environments like production or staging, give each its own secrets, and reference them from workflow YAML with `environment: <name>`. Environments referenced in YAML appear as detected entries after the repository’s next sync.'
                 }
                 action={
                   !q ? (

src/features/secrets/pages/SecretsPage.tsx

@@ -158,7 +158,7 @@ export function SecretsPage() {
                 description={
                   hasFilters
                     ? 'Nothing matches the current filters. Clear them to see every secret.'
-                    : 'Store deployment tokens, registry credentials, and API keys here — encrypted at rest, injected into jobs at run time, masked in logs.'
+                    : 'Store deployment tokens, registry credentials, and API keys here — encrypted at rest, injected into jobs at run time, masked in logs. GitHub never exposes existing secret values, so they can’t be imported: re-enter each value once and it stays encrypted here.'
                 }
                 action={
                   !hasFilters ? (