fix: second QA pass — header sniff, reshape guards, worker/DuckDB aliases

- Treat single-row CSV as headerless; warn on missing primary source
- validateReshape + reset reshape on file/sample load; safer pivot wider path
- DuckDB planner resolves duplicate source names (orders.csv (2) → orders.csv)
- Harden JSONL jq parse, explode invalid JSON, worker URL client validation
- Fix smoke-duckdb artifact (browserExecutable); clarify >1GB escalation copy
- Add reshape, duckdb alias, and single-row regression tests

Author: cursoragent

apps/worker/src/url-source.ts

@@ -12,7 +12,11 @@ function inferFormatFromName(name: string): FileFormat {
     return 'tsv';
   }
 
-  if (lower.endsWith('.jsonl') || lower.endsWith('.ndjson')) {
+  if (lower.endsWith('.ndjson')) {
+    return 'ndjson';
+  }
+
+  if (lower.endsWith('.jsonl')) {
     return 'jsonl';
   }
 

packages/core/src/duckdb-plan.ts

@@ -344,7 +344,15 @@ export function buildDuckDbQueryPlan(
   sources: DuckDbPlanSource[],
 ): DuckDbQueryPlan {
   const primarySourceName = chain.input[0]?.ref ?? sources[0]?.name ?? '';
-  const primaryIndex = sources.findIndex((source) => source.name === primarySourceName);
+  let primaryIndex = sources.findIndex((source) => source.name === primarySourceName);
+
+  if (primaryIndex < 0) {
+    primaryIndex = sources.findIndex((source) => {
+      const duplicateAlias = source.name.match(/^(.+) \((\d+)\)$/);
+
+      return duplicateAlias?.[1] === primarySourceName;
+    });
+  }
 
   if (primaryIndex < 0) {
     return {
@@ -368,6 +376,14 @@ export function buildDuckDbQueryPlan(
     registeredSources.map((source) => [source.sourceName, source.tableName]),
   );
 
+  for (const source of registeredSources) {
+    const duplicateAlias = source.sourceName.match(/^(.+) \((\d+)\)$/);
+
+    if (duplicateAlias && !tableMap.has(duplicateAlias[1]!)) {
+      tableMap.set(duplicateAlias[1]!, source.tableName);
+    }
+  }
+
   let currentSql = `SELECT * FROM ${registeredSources[primaryIndex].tableName}`;
 
   for (const step of chain.verbs) {

packages/core/src/execution.ts

@@ -876,6 +876,10 @@ export function executeVerbChain(chain: VerbChain, sources: ChainSource[]): Exec
   let current = parsedSources.get(primarySourceName) ?? { columns: [], rows: [] };
   const warnings: string[] = [];
 
+  if (primarySourceName && !parsedSources.has(primarySourceName)) {
+    warnings.push(`Primary source "${primarySourceName}" was not loaded.`);
+  }
+
   for (const step of chain.verbs) {
     switch (step.kind) {
       case 'cat':

packages/core/src/input.ts

@@ -147,10 +147,14 @@ function splitDelimitedRow(line: string, delimiter: string): string[] {
 }
 
 function inferHeader(rows: string[][]): boolean {
-  if (rows.length < 2) {
+  if (rows.length === 0) {
     return true;
   }
 
+  if (rows.length === 1) {
+    return false;
+  }
+
   const firstRow = rows[0];
   const secondRow = rows[1];
   const headerish = /^[A-Za-z_][\w .-]*$/;

packages/core/src/json-query.ts

@@ -116,8 +116,16 @@ function previewFromRows(rows: DataRow[]): PreviewTable {
 }
 
 export function applyJsonQuery(text: string, query: string): JsonQueryResult {
-  const rows = splitLines(text).map((line) => JSON.parse(line) as DataRow);
   const warnings: string[] = [];
+  const rows: DataRow[] = [];
+
+  for (const line of splitLines(text)) {
+    try {
+      rows.push(JSON.parse(line) as DataRow);
+    } catch {
+      warnings.push(`Skipped invalid JSON line: ${line.slice(0, 120)}`);
+    }
+  }
   const operations = query
     .split('|')
     .map((part) => part.trim())

packages/core/src/reshape.ts

@@ -124,8 +124,15 @@ export function explodeField(rows: DataRow[], field: string): DataRow[] {
       const trimmed = value.trim();
 
       if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
-        const parsed = JSON.parse(trimmed) as DataValue[];
-        return parsed.map((item) => ({ ...row, [field]: item }));
+        try {
+          const parsed = JSON.parse(trimmed) as DataValue[];
+
+          if (Array.isArray(parsed)) {
+            return parsed.map((item) => ({ ...row, [field]: item }));
+          }
+        } catch {
+          // Fall through to comma-split or single-row output.
+        }
       }
 
       if (trimmed.includes(',')) {
@@ -137,6 +144,56 @@ export function explodeField(rows: DataRow[], field: string): DataRow[] {
   });
 }
 
+export function validateReshape(rows: DataRow[], config: ReshapeConfig): string | null {
+  if (config.mode === 'none') {
+    return null;
+  }
+
+  if (config.mode === 'longer') {
+    const fields = parseFields(config.fields);
+
+    if (fields.length === 0) {
+      return 'Pivot longer requires at least one field name.';
+    }
+
+    if (!fields.some((field) => rows.some((row) => field in row))) {
+      return `Pivot longer fields (${fields.join(', ')}) were not found in the current preview.`;
+    }
+
+    return null;
+  }
+
+  if (config.mode === 'wider') {
+    if (config.namesFrom && rows.some((row) => config.namesFrom! in row)) {
+      return null;
+    }
+
+    const pivotFields = parseFields(config.fields);
+
+    if (!pivotFields.some((field) => rows.some((row) => field in row))) {
+      return 'Pivot wider needs a long-form column or recognizable month columns in the current preview.';
+    }
+
+    return null;
+  }
+
+  if (config.mode === 'explode') {
+    const field = config.field?.trim() ?? '';
+
+    if (!field) {
+      return 'Explode requires a field name.';
+    }
+
+    if (!rows.some((row) => field in row)) {
+      return `Explode field "${field}" was not found in the current preview.`;
+    }
+
+    return null;
+  }
+
+  return null;
+}
+
 export function applyReshape(rows: DataRow[], config: ReshapeConfig): ReshapeResult {
   let nextRows = rows;
 

packages/core/test/duckdb-plan.test.ts

@@ -139,4 +139,39 @@ describe('duckdb query planner', () => {
 
     expect(plan.supported).toBe(false);
   });
+
+  it('resolves duplicate source aliases for joins', () => {
+    const chain: VerbChain = {
+      input: [{ format: 'csv', ref: 'orders.csv' }],
+      verbs: [
+        {
+          kind: 'join',
+          opts: {
+            leftKey: 'id',
+            rightKey: 'id',
+            rightSource: 'users.csv',
+          },
+        },
+      ],
+      output: { format: 'csv' },
+    };
+
+    const plan = buildDuckDbQueryPlan(chain, [
+      {
+        dialect: null,
+        format: 'csv',
+        name: 'orders.csv (2)',
+        text: 'id,amount\n1,10\n',
+      },
+      {
+        dialect: null,
+        format: 'csv',
+        name: 'users.csv',
+        text: 'id,name\n1,Asha\n',
+      },
+    ]);
+
+    expect(plan.supported).toBe(true);
+    expect(plan.sql).toContain('LEFT JOIN input_1 AS joined_stream');
+  });
 });

packages/core/test/index.test.ts

@@ -46,6 +46,18 @@ describe('sample registry', () => {
     });
   });
 
+  it('treats a single data row as headerless', () => {
+    const dialect = sniffDialect('Alice,30\n');
+    const inspection = inspectInput(new TextEncoder().encode('Alice,30\n'), 'csv');
+
+    expect(dialect.hasHeader).toBe(false);
+    expect(inspection.preview.rows).toHaveLength(1);
+    expect(inspection.preview.rows[0]).toMatchObject({
+      column_1: 'Alice',
+      column_2: '30',
+    });
+  });
+
   it('detects BOM-based UTF-16LE content', () => {
     const bytes = new Uint8Array([0xff, 0xfe, 0x61, 0x00, 0x62, 0x00]);
 

packages/core/test/reshape.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'vitest';
+
+import { applyReshape, validateReshape } from '../src/reshape';
+
+describe('reshape validation', () => {
+  it('skips pivot wider when month columns are absent', () => {
+    const rows = [
+      { order_id: '1001', category: 'books', total: '42.5' },
+      { order_id: '1002', category: 'electronics', total: '129.99' },
+    ];
+
+    expect(
+      validateReshape(rows, {
+        mode: 'wider',
+        fields: 'jan,feb,mar',
+        namesFrom: 'month',
+        valuesFrom: 'value',
+        groupBy: 'region',
+      }),
+    ).toMatch(/long-form column|recognizable month columns/i);
+  });
+
+  it('allows pivot wider after an automatic longer step on wide-form data', () => {
+    const rows = [
+      { region: 'north', jan: '120', feb: '140' },
+      { region: 'south', jan: '98', feb: '111' },
+    ];
+
+    const longer = applyReshape(rows, {
+      mode: 'longer',
+      fields: 'jan,feb',
+      namesTo: 'month',
+      valuesTo: 'sales',
+    });
+
+    expect(
+      validateReshape(longer.rows, {
+        mode: 'wider',
+        namesFrom: 'month',
+        valuesFrom: 'sales',
+        groupBy: 'region',
+      }),
+    ).toBeNull();
+  });
+
+  it('falls back when explode field contains invalid bracket JSON', () => {
+    const rows = [{ tags: '[not valid json' }];
+
+    expect(() =>
+      applyReshape(rows, {
+        mode: 'explode',
+        field: 'tags',
+      }),
+    ).not.toThrow();
+
+    expect(
+      applyReshape(rows, {
+        mode: 'explode',
+        field: 'tags',
+      }).rows,
+    ).toHaveLength(1);
+  });
+});

packages/web/scripts/smoke-duckdb.mjs

@@ -106,22 +106,23 @@ async function runScenario(page, scenario) {
   };
 }
 
-const edgeCandidates = [
+const browserCandidates = [
+  process.env.CSVSHAPE_BROWSER_EXECUTABLE,
   process.env.CSVSHAPE_EDGE_PATH,
   'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
   'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
 ].filter(Boolean);
 
-const edgeExecutable = edgeCandidates.find((candidate) => existsSync(candidate));
+const browserExecutable = browserCandidates.find((candidate) => existsSync(candidate));
 
-if (!edgeExecutable) {
-  throw new Error('Microsoft Edge executable was not found for the DuckDB-WASM smoke test.');
-}
-
-const browser = await chromium.launch({
-  executablePath: edgeExecutable,
-  headless: true,
-});
+const browser = browserExecutable
+  ? await chromium.launch({
+      executablePath: browserExecutable,
+      headless: true,
+    })
+  : await chromium.launch({
+      headless: true,
+    });
 
 const consoleMessages = [];
 let page;
@@ -156,7 +157,7 @@ try {
       {
         baseUrl,
         consoleMessages,
-        edgeExecutable,
+        browserExecutable: browserExecutable ?? null,
         results,
         status: 'ok',
       },
@@ -172,7 +173,7 @@ try {
         artifactPath,
         baseUrl,
         consoleMessages,
-        edgeExecutable,
+        browserExecutable: browserExecutable ?? null,
         results,
         status: 'ok',
       },