Skip to content

Commit d391f85

Browse files
committed
Merge remote-tracking branch 'origin/main' into upstream/apikeys-privilege-1833
2 parents 38c1686 + 3ad69e9 commit d391f85

30 files changed

Lines changed: 6614 additions & 270 deletions

‎frontend/e2e/dashboard-editor.spec.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ async function addField(editor: Locator, label: string, value: string) {
9090
test('creating every chart type through the editor saves classic-compatible keys', async ({
9191
page,
9292
}) => {
93+
// Eleven editor flows and four full-page accessibility scans share this budget.
94+
test.setTimeout(60_000);
95+
9396
const server = await serve(page);
9497
const sql = 'SELECT time, service, requests FROM "app-logs"';
9598

‎frontend/src/features/logs/SavedViews.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@ describe('server saved views', () => {
8787
]);
8888
expect(available.map(({ decoded }) => decoded?.dataset)).toEqual(['app', 'app', undefined]);
8989
expect(savedLogViewsForDataset([canonical], 'sql')).toEqual([]);
90+
const promql = {
91+
...classic,
92+
filter_name: 'Request rate',
93+
query: { filter_type: 'promql' as const, filter_query: 'rate(requests_total[5m])' },
94+
};
95+
expect(savedLogViewsForDataset([promql], 'app')).toEqual([]);
9096
});
9197
it('rejects classic formats and mismatched dataset metadata', () => {
9298
const { explorer_state: _state, ...classic } = savedLogViewBody(state, 'Classic');

‎frontend/src/features/logs/SavedViews.tsx‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,11 @@ export function savedLogViewsForDataset(views: SavedLogView[], dataset: string)
2525
return decoded.dataset === dataset ? [{ view, decoded }] : [];
2626
} catch {
2727
// Classic dataset-scoped views remain visible, but canonical views need a
28-
// valid explorer state to establish which dataset they belong to.
29-
return view.stream_name !== 'sql' && view.stream_name === dataset
28+
// valid explorer state to establish which dataset they belong to. Saved
29+
// PromQL queries share the endpoint but are not log views.
30+
return view.query.filter_type !== 'promql' &&
31+
view.stream_name !== 'sql' &&
32+
view.stream_name === dataset
3033
? [{ view, decoded: undefined }]
3134
: [];
3235
}

‎frontend/src/features/sql/serverSavedQueries.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,17 @@ const classicSearch: SavedLogView = {
9292
};
9393

9494
describe('server saved SQL queries', () => {
95+
it('ignores saved PromQL filters', () => {
96+
const promql: SavedLogView = {
97+
...classicSearch,
98+
filter_id: '01HPROMQL',
99+
stream_name: 'otel_metrics',
100+
query: { filter_type: 'promql', filter_query: 'sum(rate(http_requests_total[5m]))' },
101+
};
102+
expect(savedLogView(promql)).toBe(true);
103+
expect(serverSavedQuery(promql)).toBeUndefined();
104+
expect(serverSavedQueries([promql])).toEqual([]);
105+
});
95106
it('accepts the real classic fixtures through the client contract', () => {
96107
for (const view of [classicSql, classicLogsSql, nativeLogsSql, classicBuilder, classicSearch])
97108
expect(savedLogView(view)).toBe(true);

‎frontend/src/lib/logsContract.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,17 @@ describe('saved view HTTP contract', () => {
106106
: client.convertLogViewToSql('id'),
107107
).rejects.toBeInstanceOf(ApiError);
108108
});
109+
it('accepts saved PromQL filters in the shared list without failing log views', async () => {
110+
const promql: SavedLogView = {
111+
...body,
112+
filter_id: 'promql-id',
113+
stream_name: 'otel_metrics',
114+
query: { filter_type: 'promql', filter_query: 'sum(rate(http_requests_total[5m]))' },
115+
};
116+
const saved = { ...body, filter_id: 'server-id' };
117+
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(json([saved, promql])));
118+
expect(await createClient({ mode: 'live' }).listLogViews()).toEqual([saved, promql]);
119+
});
109120
it('validates optional explorer state while retaining classic metadata', () => {
110121
for (const explorer_state of [undefined, null, 'encoded'])
111122
expect(savedLogView({ ...body, explorer_state, classic_metadata: { anything: true } })).toBe(

‎frontend/src/lib/logsContract.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ export function savedLogView(value: unknown): value is SavedLogView {
4040
optionalString(value.explorer_state) &&
4141
object(value.query) &&
4242
typeof value.query.filter_type === 'string' &&
43-
['filter', 'sql', 'search'].includes(value.query.filter_type) &&
43+
['filter', 'sql', 'search', 'promql'].includes(value.query.filter_type) &&
4444
optionalString(value.query.filter_query) &&
4545
(value.query.filter_builder == null || builder(value.query.filter_builder)) &&
4646
(value.time_filter == null ||

‎frontend/src/lib/types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ export type SavedLogView = {
130130
filter_name: string;
131131
stream_name: string;
132132
query: {
133-
filter_type: 'filter' | 'sql' | 'search';
133+
filter_type: 'filter' | 'sql' | 'search' | 'promql';
134134
filter_query?: string | null;
135135
filter_builder?: SavedLogViewBuilder | null;
136136
};
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__pycache__/
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Ingestion performance and compatibility checks
2+
3+
These scripts launch disposable local-store servers on loopback ports. They require two already-built Parseable binaries, Python 3, Linux `/proc`, and `taskset` when CPU affinity is supplied. They do not connect to an existing deployment. Each output directory must be new. Test credentials are local dummy values; telemetry and update checks are disabled.
4+
5+
Build the baseline and candidate with the same Rust toolchain, lockfile, features, allocator, profile and real UI assets. Preserve the baseline executable before building the candidate. Run builds and other experiments before timing; avoid competing activity on the test host.
6+
7+
```sh
8+
python scripts/ingest-performance/compatibility.py --baseline /abs/baseline --candidate /abs/candidate --output /abs/results/compat --cpus 2,4,6,8
9+
python scripts/ingest-performance/run.py --baseline /abs/baseline --candidate /abs/candidate --output /abs/results/burst --pairs 4 --cpus 2,4,6,8
10+
python scripts/ingest-performance/sustained.py --baseline /abs/baseline --candidate /abs/candidate --output /abs/results/sustained --pairs 2 --seconds 300 --requests-per-second 20 --cpus 2,4,6,8
11+
python scripts/ingest-performance/mixed.py --baseline /abs/baseline --candidate /abs/candidate --output /abs/results/mixed --pairs 2 --seconds 75 --requests-per-second 2 --queries-per-second 2 --cpus 2,4,6,8
12+
```
13+
14+
Select available physical cores on your machine, or pass `--cpus ''` to omit affinity. Four Actix and four Rayon workers are used in both arms. The burst runner alternates arm order across pairs; `--smoke` runs a small functional check. Sustained and mixed runners also alternate arm order with `--pairs`; the default remains one pair. The offered load starts at wall-clock minute :05 after warmup, unless phase alignment is disabled; its actual anchored start is recorded. Their fixed offered rates do not measure maximum sustainable capacity. Use separate comparisons against the original baseline and reviewed PR binary to isolate the cost of remediation; optional `--reviewed` adds a third arm but ABC/CBA ordering does not balance the middle arm across positions.
15+
16+
The mixed runner ingests 8,204-row, 127-field scalar requests while querying the same dataset. Its default body is 9,556,551 bytes, checked before starting servers against the 10 MiB HTTP limit. The server adds `p_timestamp`; production queries sort fields and exercise reordered schema adaptation during capture. Each concurrent query must include every request acknowledged before it starts and cannot include requests that start after it finishes; in-flight rows remain allowed. These timestamp bounds are checked after load and saved per query. Exact request/row identity is checked after load, drain and persisted-only restart. Mixed CPU includes ingestion and queries. Query latency is not isolated capture time or lock-hold time.
17+
18+
`--diagnostics` streams bounded 100 ms thread state/wchan/schedstat/fault counters, process I/O, host pressure and swap counters. Both arms receive the same instrumentation. Collection cost, errors, truncation and the byte cap are reported; use matched on/off runs before attributing timing changes to the server. A clean run or an off-CPU sample cannot explain a historical stall. The diagnostic does not enumerate unrelated host processes or identify their I/O. Client lag, workload duration, live payload bytes and sampled staging bytes are bounded; hitting a safety bound invalidates the run rather than demonstrating a performance threshold.
19+
20+
Every timed request must succeed. Request IDs are unique, including warmup requests. Verification checks each request's row count, distinct row IDs and bounds. Burst checks repeat after SIGKILL and restart, polling for eventual query visibility for up to 120 iterations. This tests process-crash recovery, not power-loss durability or immediate recovery readiness.
21+
22+
The sustained runner samples server CPU/RSS and file sizes each second. After load stops, it polls the target dataset’s data-bearing staging files for up to 240 seconds, requires three consecutive empty observations, and verifies rows again. It then kills its server, preserves the old staging directory and starts with empty staging to check rows from the persisted local store. Internal `pstats` work may continue after the target dataset drains. CPU is reported separately for the ACK window and through observed drain (the latter also includes the first verification query). Sampled peak RSS includes verification queries and drain; it is not an ingestion-only peak or allocator high-water mark. Staged file size is not a total in-memory backlog measure.
23+
24+
Burst payloads are prepared before timing and capped at 768 MiB of client payload data. The current app/wide workloads retain about 413/528 MB in the client; reported server RSS excludes the client, kernel page cache and other processes. Request identities add one string field to each row. Payloads, binary hashes, timing phase, configuration and host load are recorded. Short burst results may overlap periodic background work differently; inspect all repetitions rather than a single ratio.
25+
26+
The compatibility corpus compares HTTP status/error responses, schemas, ingestion accounting and logical row multisets. It deliberately ignores generated `p_timestamp` and unspecified SQL result order. Every request's expected status is asserted independently, including intended rejections and raw OTLP cases; `partitiontypes` pins one named inherited 500 with its known one-row partial outcome. Successful setup/readback statuses, row counts and accounting are asserted, and the static case additionally checks schema types, IDs and decoded values. Equal errors cannot satisfy these success checks. Exact non-static content/error bodies still rely on differential comparison. Ordered flattening, exact RecordBatch/schema behavior and failure boundaries are covered separately by the Rust differential tests.
27+
28+
Run the server-free oracle and harness tests with `python3 -B -m unittest discover -s scripts/ingest-performance -p 'test_*.py' -v`. `compatibility-baseline.json` is the real successful normalized transcript from commit `6aaee1d61c261c6b71c0dad30b6bee75ac654cab`. Replay tests reject equal 503s on ingests/readbacks and incorrect static values, schemas, or accounting; a replay pass does not replace live validation on the final binaries.
29+
30+
To render PNG/SVG charts, install matplotlib and numpy in an isolated environment, then:
31+
32+
```sh
33+
python scripts/ingest-performance/plot.py --burst /abs/results/burst --sustained /abs/results/sustained --output /abs/results/figures
34+
```
35+
36+
The local-store results do not establish S3 ingestion capacity. Multipart memory and failure behavior require separate storage probes. Keep exact build provenance and limitations with any reported figures.

0 commit comments

Comments
 (0)