Skip to content

Commit 85675fc

Browse files
knudttyknudtty
authored
feat: stream filter keys and value results to UI (#3348)
## Summary Filter queries no longer wait for all values to arrive. Values are streamed and the UI updates as keys and values arrive. It now behaves that: 1. Keys are fetched 2. Keys are returned. The UI updates and shows keys even if they have not yet fetched values. 3. Values are fetched and begin streaming in. The filters UI updates as new values arrive. 4. Eventually all value queries finish. The filters UI shows the finished state. ### Screenshots or video #### Before https://github.com/user-attachments/assets/4ac736e6-53e6-4292-8f79-564e66ce9bcb #### After https://github.com/user-attachments/assets/5d2037be-2260-4cfe-9c42-dcbf1e0f082a ### How to test on Vercel preview **Steps:** 1. Go to logs 2. Assign a longer time range (1-2 days) 3. View the filters bar ### References - Linear Issue:
1 parent 685f465 commit 85675fc

16 files changed

Lines changed: 1103 additions & 198 deletions

File tree

‎.changeset/stream-filter-values.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@hyperdx/app': patch
3+
'@hyperdx/common-utils': patch
4+
---
5+
6+
perf: stream filter values into the UI as each query finishes instead of
7+
waiting for all of them

‎packages/app/src/components/DBSearchPageFilters.tsx‎

Lines changed: 53 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ import {
8383
} from './DBSearchPageFilters/PinShareMenu';
8484
import { SharedFiltersSection } from './DBSearchPageFilters/SharedFilters';
8585
import {
86+
type FacetEntry,
8687
getFilterStateEntry,
8788
groupFacetsByBaseName,
8889
toQuotedClickHouseKeyExpression,
@@ -1016,7 +1017,7 @@ export const FilterGroup = ({
10161017
chevron: 'm-0',
10171018
label: 'p-0',
10181019
}}
1019-
className={hasOptions ? '' : 'opacity-50'}
1020+
className={hasOptions || optionsLoading ? '' : 'opacity-50'}
10201021
>
10211022
<Tooltip
10221023
openDelay={displayName.length > 26 ? 0 : 1500}
@@ -1028,12 +1029,21 @@ export const FilterGroup = ({
10281029
>
10291030
<Text size="xs" fw="500" truncate="end">
10301031
{displayName}
1031-
{showFilterCounts && (
1032-
<Text
1033-
component="span"
1034-
size="xs"
1035-
c="dimmed"
1036-
>{` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`}</Text>
1032+
{totalAppliedFiltersSize === 0 && optionsLoading ? (
1033+
<Loader
1034+
size={8}
1035+
color="var(--color-text-muted)"
1036+
ml={6}
1037+
data-testid={`filter-values-loading-${name}`}
1038+
/>
1039+
) : (
1040+
showFilterCounts && (
1041+
<Text
1042+
component="span"
1043+
size="xs"
1044+
c="dimmed"
1045+
>{` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`}</Text>
1046+
)
10371047
)}
10381048
</Text>
10391049
</Tooltip>
@@ -1232,7 +1242,7 @@ const DBSearchPageFiltersComponent = ({
12321242
filterState,
12331243
showMoreFields,
12341244
});
1235-
const facets = fetchFacetsData.keyValues;
1245+
const { keyValues: facets, pendingKeys } = fetchFacetsData;
12361246

12371247
useEffect(() => {
12381248
if (error) {
@@ -1246,7 +1256,7 @@ const DBSearchPageFiltersComponent = ({
12461256
}, [error]);
12471257

12481258
// Merge pinned filter values into the queried facets, so that pinned values are always available
1249-
const facetsWithPinnedValues = useMemo(() => {
1259+
const facetsWithPinnedValues = useMemo((): FacetEntry[] => {
12501260
const facetsMap = new Map((facets ?? []).map(f => [f.key, f.value]));
12511261
const mergedKeys = new Set<string>([
12521262
...facetsMap.keys(),
@@ -1262,9 +1272,13 @@ const DBSearchPageFiltersComponent = ({
12621272
...(pinnedValues ?? []),
12631273
]);
12641274

1265-
return { key, value: Array.from(mergedValues) };
1275+
return {
1276+
key,
1277+
value: Array.from(mergedValues),
1278+
isLoading: pendingKeys.has(key),
1279+
};
12661280
});
1267-
}, [facets, pinnedFilters, getPinnedFields]);
1281+
}, [facets, pinnedFilters, getPinnedFields, pendingKeys]);
12681282

12691283
// Build the set of team-pinned fields for the Shared Filters section,
12701284
// so we can avoid duplicating them in the regular Filters list below.
@@ -1278,7 +1292,7 @@ const DBSearchPageFiltersComponent = ({
12781292

12791293
// Build the facet list for the Shared Filters section.
12801294
// For each team-pinned field: merge pinned values with dynamic facet values.
1281-
const sharedFacets = useMemo(() => {
1295+
const sharedFacets = useMemo((): FacetEntry[] => {
12821296
if (sharedFilterKeys.size === 0) return [];
12831297

12841298
const facetMap = new Map(
@@ -1288,6 +1302,7 @@ const DBSearchPageFiltersComponent = ({
12881302
return Array.from(sharedFilterKeys).map(key => {
12891303
const teamVals = pinnedFiltersApiData?.team?.filters[key] ?? [];
12901304
const dynamicValues = facetMap.get(key) ?? [];
1305+
const isLoading = pendingKeys.has(key);
12911306

12921307
let merged: (string | boolean)[];
12931308
if (teamVals.length > 0) {
@@ -1302,12 +1317,17 @@ const DBSearchPageFiltersComponent = ({
13021317
merged = [...dynamicValues];
13031318
}
13041319

1305-
return { key, value: merged };
1320+
return { key, value: merged, isLoading };
13061321
});
1307-
}, [sharedFilterKeys, facetsWithPinnedValues, pinnedFiltersApiData]);
1322+
}, [
1323+
sharedFilterKeys,
1324+
facetsWithPinnedValues,
1325+
pinnedFiltersApiData,
1326+
pendingKeys,
1327+
]);
13081328

13091329
const shownFacets = useMemo(() => {
1310-
const _facets: { key: string; value: (string | boolean)[] }[] = [];
1330+
const _facets: FacetEntry[] = [];
13111331
for (const _facet of facetsWithPinnedValues ?? []) {
13121332
const facet = structuredClone(_facet);
13131333
if (jsonColumns?.some(col => facet.key.startsWith(col))) {
@@ -1319,12 +1339,18 @@ const DBSearchPageFiltersComponent = ({
13191339
continue;
13201340
}
13211341

1322-
// don't include empty facets, unless they are already selected or pinned
1342+
// don't include empty facets, unless they are already selected, pinned,
1343+
// or still waiting on their values
13231344
const filter = filterState[facet.key];
13241345
const hasSelectedValues =
13251346
filter && (filter.included.size > 0 || filter.excluded.size > 0);
13261347
const isPinned = isFieldPinned(facet.key);
1327-
if (facet.value?.length > 0 || hasSelectedValues || isPinned) {
1348+
if (
1349+
facet.value?.length > 0 ||
1350+
hasSelectedValues ||
1351+
isPinned ||
1352+
facet.isLoading
1353+
) {
13281354
_facets.push(facet);
13291355
}
13301356
}
@@ -1466,7 +1492,7 @@ const DBSearchPageFiltersComponent = ({
14661492
*/
14671493
const renderFacetList = useCallback(
14681494
(
1469-
facets: { key: string; value: (string | boolean)[] }[],
1495+
facets: FacetEntry[],
14701496
options?: { keyPrefix?: string; isDefaultExpanded?: boolean },
14711497
) => {
14721498
const { keyPrefix = '', isDefaultExpanded: forceExpanded } =
@@ -1582,7 +1608,7 @@ const DBSearchPageFiltersComponent = ({
15821608
value,
15831609
label: value.toString(),
15841610
}))}
1585-
optionsLoading={isFacetsLoading}
1611+
optionsLoading={isFacetsLoading || !!facet.isLoading}
15861612
selectedValues={
15871613
getFilterStateEntry(filterState, facet.key) ?? {
15881614
included: new Set(),
@@ -1876,16 +1902,15 @@ const DBSearchPageFiltersComponent = ({
18761902
/>
18771903
)}
18781904

1879-
{isFacetsLoading ? (
1880-
<Flex align="center" justify="center">
1881-
<Loader size="xs" color="gray" />
1882-
</Flex>
1883-
) : (
1884-
shownFacets.length === 0 && (
1905+
{shownFacets.length === 0 &&
1906+
(isFacetsLoading ? (
1907+
<Flex align="center" justify="center">
1908+
<Loader size="xs" color="gray" />
1909+
</Flex>
1910+
) : (
18851911
<Text size="xxs">No filters available</Text>
1886-
)
1887-
)}
1888-
{/* Show facets even when loading to ensure pinned filters are visible while loading */}
1912+
))}
1913+
{/* Filters show their own loader until their values arrive */}
18891914
{renderFacetList(shownFacets)}
18901915

18911916
<Button

‎packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,20 @@ import {
88
FILTER_SEARCH_MIN_ITEMS,
99
FilterSearchInput,
1010
} from '@/components/DBSearchPageFilters/FilterSearchInput';
11+
import type { FacetEntry } from '@/components/DBSearchPageFilters/utils';
1112
import { isColumnInSelect } from '@/utils';
1213

1314
import classes from '@styles/SearchPage.module.scss';
1415

1516
type NestedFilterGroupProps = {
1617
name: string;
17-
childFilters: {
18-
key: string;
19-
value: (string | boolean)[];
18+
childFilters: (FacetEntry & {
2019
propertyPath: string;
2120
// Canonical (quoted/bracket) SQL form of `key`, used wherever the key
2221
// becomes a raw SQL expression (distribution query, "Add column" SELECT).
2322
// Falls back to `key` when absent.
2423
sqlKey?: string;
25-
}[];
24+
})[];
2625
selectedValues?: FilterState;
2726
onChange: (key: string, value: string | boolean) => void;
2827
onClearClick: (key: string) => void;
@@ -258,7 +257,7 @@ export const NestedFilterGroup = ({
258257
value: value,
259258
label: value.toString(),
260259
}))}
261-
optionsLoading={false}
260+
optionsLoading={child.isLoading ?? false}
262261
selectedValues={childSelectedValues}
263262
onChange={value => onChange(child.key, value)}
264263
onClearClick={() => onClearClick(child.key)}

‎packages/app/src/components/DBSearchPageFilters/hooks.test.tsx‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
1010
import { act, renderHook, waitFor } from '@testing-library/react';
1111

1212
import api from '@/api';
13+
import type { Facet } from '@/hooks/useMetadata';
1314
import * as useMetadataModule from '@/hooks/useMetadata';
1415
import * as searchFiltersModule from '@/searchFilters';
1516
import * as sourceModule from '@/source';
@@ -68,6 +69,7 @@ jest.mock('@/hooks/useMetadata', () => ({
6869
useMapColumns: jest.fn(),
6970
useAllFields: jest.fn(),
7071
useGetKeyValues: jest.fn(),
72+
useFilterKeysFetchLimit: jest.fn(),
7173
}));
7274

7375
const useMe = jest.mocked(api.useMe);
@@ -82,6 +84,9 @@ const useJsonColumns = jest.mocked(useMetadataModule.useJsonColumns);
8284
const useMapColumns = jest.mocked(useMetadataModule.useMapColumns);
8385
const useAllFields = jest.mocked(useMetadataModule.useAllFields);
8486
const useGetKeyValues = jest.mocked(useMetadataModule.useGetKeyValues);
87+
const useFilterKeysFetchLimit = jest.mocked(
88+
useMetadataModule.useFilterKeysFetchLimit,
89+
);
8590

8691
const CHART_CONFIG: BuilderChartConfigWithDateRange = {
8792
connection: 'conn1',
@@ -121,6 +126,7 @@ const makeLogSource = (opts: { withMVs: boolean }) => ({
121126

122127
type SourceQueryResult = ReturnType<typeof sourceModule.useSource>;
123128
type AllFieldsResult = ReturnType<typeof useMetadataModule.useAllFields>;
129+
type KeyValuesResult = ReturnType<typeof useMetadataModule.useGetKeyValues>;
124130
type MetadataWithSettings = ReturnType<
125131
typeof useMetadataModule.useMetadataWithSettings
126132
>;
@@ -192,6 +198,8 @@ function setupDefaultMocks({ withMVs }: { withMVs: boolean }) {
192198
isFetching: false,
193199
error: null,
194200
} as any);
201+
202+
useFilterKeysFetchLimit.mockReturnValue(100);
195203
}
196204

197205
describe('useFetchFacets', () => {
@@ -479,6 +487,88 @@ describe('useFetchFacets', () => {
479487
});
480488
});
481489

490+
describe('pending keys', () => {
491+
const TWO_FIELDS = [
492+
{
493+
path: ['ServiceName'],
494+
type: 'LowCardinality(String)',
495+
jsType: 'string',
496+
},
497+
{
498+
path: ['SeverityText'],
499+
type: 'LowCardinality(String)',
500+
jsType: 'string',
501+
},
502+
];
503+
504+
const mockKeyValues = (data: Facet[] | undefined, isFetching: boolean) =>
505+
useGetKeyValues.mockReturnValue({
506+
data,
507+
isLoading: false,
508+
isFetching,
509+
error: null,
510+
} as unknown as KeyValuesResult);
511+
512+
const renderAllMode = () =>
513+
renderHook(
514+
() =>
515+
useFetchFacets({
516+
chartConfig: CHART_CONFIG,
517+
sourceId: 'source1',
518+
dateRange: DATE_RANGE,
519+
mode: 'all',
520+
}),
521+
{ wrapper: makeWrapper().wrapper },
522+
);
523+
524+
it('lists keys whose values have not arrived yet as empty, pending facets', () => {
525+
setupDefaultMocks({ withMVs: false });
526+
useAllFields.mockReturnValue({
527+
data: TWO_FIELDS,
528+
} as unknown as AllFieldsResult);
529+
mockKeyValues([{ key: 'SeverityText', value: ['info'] }], true);
530+
531+
const { result } = renderAllMode();
532+
533+
expect(result.current.data.keyValues).toEqual([
534+
{ key: 'ServiceName', value: [] },
535+
{ key: 'SeverityText', value: ['info'] },
536+
]);
537+
expect(result.current.data.pendingKeys).toEqual(new Set(['ServiceName']));
538+
});
539+
540+
it('drops keys without values once the values query finishes', () => {
541+
setupDefaultMocks({ withMVs: false });
542+
useAllFields.mockReturnValue({
543+
data: TWO_FIELDS,
544+
} as unknown as AllFieldsResult);
545+
mockKeyValues([{ key: 'SeverityText', value: ['info'] }], false);
546+
547+
const { result } = renderAllMode();
548+
549+
expect(result.current.data.keyValues).toEqual([
550+
{ key: 'SeverityText', value: ['info'] },
551+
]);
552+
expect(result.current.data.pendingKeys.size).toBe(0);
553+
});
554+
555+
it('does not list keys past the team key fetch limit', () => {
556+
setupDefaultMocks({ withMVs: false });
557+
useAllFields.mockReturnValue({
558+
data: TWO_FIELDS,
559+
} as unknown as AllFieldsResult);
560+
useFilterKeysFetchLimit.mockReturnValue(1);
561+
mockKeyValues(undefined, true);
562+
563+
const { result } = renderAllMode();
564+
565+
expect(result.current.data.keyValues).toEqual([
566+
{ key: 'ServiceName', value: [] },
567+
]);
568+
expect(result.current.data.pendingKeys).toEqual(new Set(['ServiceName']));
569+
});
570+
});
571+
482572
describe('loadMoreFacetsForKey (raw-tables pipeline)', () => {
483573
it('reports the key as loading while the fetch is in flight, then clears it', async () => {
484574
setupDefaultMocks({ withMVs: false });

0 commit comments

Comments
 (0)