Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/quiet-filters-hide.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@hyperdx/app': patch
'@hyperdx/common-utils': patch
---

fix: Show distribution only labels a value "<1%" when the sample shows it is rare, not when it has no matching rows, and shows 100% when the field's filter allows only that value
44 changes: 37 additions & 7 deletions packages/app/src/components/DBSearchPageFilters.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ const INITIAL_MAX_VALUES_DISPLAYED = 10;
/* The maximum number of values per filter to render at once after loading more */
const SHOW_MORE_MAX_VALUES_DISPLAYED = 50;

/* The 100 most common values are enough to find any value present in at least 1% of rows */
const DISTRIBUTION_VALUE_LIMIT = 100;

// This function will clean json string attributes specifically. It will turn a string like
// 'toString(ResourceAttributes.`hdx`.`sdk`.`version`)' into 'ResourceAttributes.hdx.sdk.version'.
export function cleanedFacetName(key: string): string {
Expand Down Expand Up @@ -133,6 +136,7 @@ type FilterCheckboxProps = {
className?: string;
percentage?: number;
isPercentageLoading?: boolean;
isPercentageExact?: boolean;
};

const TextButton = ({
Expand Down Expand Up @@ -165,19 +169,24 @@ const TextButton = ({
type FilterPercentageProps = {
percentage: number;
isLoading?: boolean;
// Share is guaranteed by the filter rather than estimated from the sample.
isExact?: boolean;
'data-testid'?: string;
};

const FilterPercentage = ({
percentage,
isLoading,
isExact,
'data-testid': dataTestId,
}: FilterPercentageProps) => {
const formattedPercentage =
percentage < 1
? `<1%`
: percentage >= 99.5
? `>99%`
? isExact
? '100%'
: `>99%`
: `~${Math.round(percentage)}%`;

return (
Expand Down Expand Up @@ -205,6 +214,7 @@ const FilterCheckbox = ({
className,
percentage,
isPercentageLoading,
isPercentageExact,
}: FilterCheckboxProps) => {
const [pinMenuOpened, setPinMenuOpened] = useState(false);
const testIdPrefix = `filter-checkbox-${columnName}-${label}`;
Expand Down Expand Up @@ -268,6 +278,7 @@ const FilterCheckbox = ({
<FilterPercentage
percentage={percentage}
isLoading={isPercentageLoading}
isExact={isPercentageExact}
data-testid={`filter-distribution-${columnName}-${label}`}
/>
)}
Expand Down Expand Up @@ -492,7 +503,8 @@ const FilterGroupBody = ({
{
chartConfig: { ...chartConfig, dateRange },
key: distributionKey || name,
limit: 100, // The 100 most common values are enough to find any values that are present in at least 1% of rows
// One extra row tells a capped result apart from exactly 100 values
limit: DISTRIBUTION_VALUE_LIMIT + 1,
},
{
enabled: showDistributions,
Expand Down Expand Up @@ -629,6 +641,22 @@ const FilterGroupBody = ({
[onChange, selectedValues],
);

// The distribution query runs under this field's own filter, so a value the
// filter rules out has zero rows; showing it as "<1%" would read as rare.
const isExcludedByOwnFilter = (value: string | boolean) =>
selectedValues.excluded.has(value) ||
(selectedValues.included.size > 0 && !selectedValues.included.has(value));

const getPercentage = (value: string | boolean) => {
if (!showDistributions || !distributionData || isExcludedByOwnFilter(value))
return undefined;
const percentage = distributionData.get(value.toString());
if (percentage != null) return percentage;
// Missing from a capped result means it ranks below the 100th value, so
// under 1%. Missing from an uncapped result means it wasn't seen at all.
return distributionData.size > DISTRIBUTION_VALUE_LIMIT ? 0 : undefined;
Comment on lines +655 to +657

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Rare values lose their label When a value appears in the facet options but its rare rows are missed by the separate distribution sample, a result with at most 100 entries makes this code show no percentage. That hides the <1% label for a value that is present: an uncapped sample shows only that the value was not sampled, not that it has no matching rows.

Fix in Claude Code Fix in Conductor Fix in Cursor Fix in Codex

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 major — Rare values that really exist lose their "<1%" label on sampled tables

getValuesDistribution (packages/common-utils/src/core/metadata.ts:2362-2377) samples roughly 100k rows once a table is larger than that. So a value missing from a result that wasn't cut off may simply not have been picked by the sample; it doesn't mean the value is absent. The facet options come from getKeyValues over the same filtered chartConfig, so an unselected option in the list does have matching rows. Example: a 10M-row table where a listed status value appears in 0.001% of rows. The sample misses it, and it now shows no label, which looks like 'no rows' even though clicking it returns results. Before this change it showed a correct <1%. Suggested fix: once isExcludedByOwnFilter has ruled out filtered values, fall back to 0 (<1%) for any other missing value. If you want to keep the 'not seen' case, only return undefined when the query wasn't sampled (have getValuesDistribution also return sample_factor, and check it is 1).

Nothing blocks merge automatically, but a maintainer will expect this fixed if it is a real defect in code this PR changes. If it is about surrounding code, reply and say so instead of patching. Do not widen the PR. How to respond

};

const isLimitingDisplayedItems =
sortedMatchingOptions.length > displayedOptions.length;

Expand Down Expand Up @@ -682,11 +710,11 @@ const FilterGroupBody = ({
onSharedPinClick ? () => onSharedPinClick(option.value) : undefined
}
isPercentageLoading={isFetchingDistribution}
percentage={
showDistributions && distributionData
? (distributionData.get(option.value.toString()) ?? 0)
: undefined
isPercentageExact={
selectedValues.included.size === 1 &&
selectedValues.included.has(option.value)
}
percentage={getPercentage(option.value)}
/>
))}
{optionsLoading ? (
Expand Down Expand Up @@ -799,7 +827,9 @@ function FilterGroupActions({
<>
<Tooltip
label={
showDistributions ? 'Hide Distribution' : 'Show Distribution'
showDistributions
? 'Hide distribution'
: 'Show distribution (estimated from a sample of rows)'
}
position="top"
withArrow
Expand Down
150 changes: 139 additions & 11 deletions packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ jest.mock('@/hooks/useMetadata', () => ({
.mockReturnValue({ data: undefined, isFetching: false, error: undefined }),
}));

const mockDistribution = (data: Map<string, number>) =>
jest.mocked(useGetValuesDistribution).mockReturnValue({
data,
isFetching: false,
error: null,
} as UseQueryResult<Map<string, number>>);

describe('cleanedFacetName', () => {
describe('basic functionality', () => {
it('should return non-toString strings unchanged', () => {
Expand Down Expand Up @@ -441,15 +448,13 @@ describe('FilterGroup', () => {
});

it('should show selected items first, then sort by counts, if percentages when they are enabled', () => {
jest.mocked(useGetValuesDistribution).mockReturnValue({
data: new Map([
mockDistribution(
new Map([
['apple', 30],
['banana', 20],
['zebra', 50],
]),
isFetching: false,
error: null,
} as UseQueryResult<Map<string, number>>);
);

renderWithMantine(
<FilterGroup
Expand All @@ -470,14 +475,12 @@ describe('FilterGroup', () => {
});

it('should show percentages, if enabled', async () => {
jest.mocked(useGetValuesDistribution).mockReturnValue({
data: new Map([
mockDistribution(
new Map([
['apple', 99.2],
['zebra', 0.6],
]),
isFetching: false,
error: null,
} as UseQueryResult<Map<string, number>>);
);

renderWithMantine(
<FilterGroup
Expand All @@ -497,9 +500,134 @@ describe('FilterGroup', () => {
const options = screen.getAllByTestId(/filter-checkbox-.+-input/);
expect(options).toHaveLength(3);
const labels = screen.getAllByText(/%/);
expect(labels).toHaveLength(2);
expect(labels[0]).toHaveTextContent('~99%'); // apple
expect(labels[1]).toHaveTextContent('<1%'); // zebra
expect(labels[2]).toHaveTextContent('<1%'); // banana
// banana is missing from an uncapped result, so it was not seen at all
});

it('should show <1% for a value missing from a capped result', async () => {
mockDistribution(
new Map(
Array.from({ length: 101 }, (_, i) => [`value-${i}`, 1] as const),
),
);

renderWithMantine(<FilterGroup {...defaultProps} />);

await userEvent.click(
screen.getByTestId('toggle-distribution-button-Test Filter'),
);

expect(
screen.getByTestId('filter-distribution-Test Filter-banana'),
).toHaveTextContent('<1%');
});

it("should run the distribution query under the field's own filter", async () => {
const filters = [
{ type: 'sql' as const, condition: "Test Filter IN ('apple')" },
];
renderWithMantine(
<FilterGroup
{...defaultProps}
chartConfig={{ ...defaultProps.chartConfig, filters }}
selectedValues={{
included: new Set(['apple']),
excluded: new Set(),
}}
/>,
);

await userEvent.click(
screen.getByTestId('toggle-distribution-button-Test Filter'),
);

expect(jest.mocked(useGetValuesDistribution)).toHaveBeenLastCalledWith(
expect.objectContaining({
chartConfig: expect.objectContaining({ filters }),
}),
expect.objectContaining({ enabled: true }),
);
});

it('should show 100% for the only value the field filter allows, and nothing for the rest', async () => {
mockDistribution(new Map([['apple', 100]]));

renderWithMantine(
<FilterGroup
{...defaultProps}
selectedValues={{
included: new Set(['apple']),
excluded: new Set(),
}}
/>,
);
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

await userEvent.click(
screen.getByTestId('toggle-distribution-button-Test Filter'),
);

const labels = screen.getAllByText(/%/);
expect(labels).toHaveLength(1);
expect(labels[0]).toHaveTextContent('100%');
});

it('should keep estimated percentages when the field filter allows several values', async () => {
mockDistribution(
new Map([
['apple', 99.7],
['banana', 0.3],
]),
);

renderWithMantine(
<FilterGroup
{...defaultProps}
selectedValues={{
included: new Set(['apple', 'banana']),
excluded: new Set(),
}}
/>,
);

await userEvent.click(
screen.getByTestId('toggle-distribution-button-Test Filter'),
);

expect(
screen.getByTestId('filter-distribution-Test Filter-apple'),
).toHaveTextContent('>99%');
});

it('should not show a percentage for an excluded value', async () => {
mockDistribution(
new Map([
['apple', 60],
['banana', 40],
]),
);

renderWithMantine(
<FilterGroup
{...defaultProps}
selectedValues={{
included: new Set(),
excluded: new Set(['zebra']),
}}
/>,
);

await userEvent.click(
screen.getByTestId('toggle-distribution-button-Test Filter'),
);

expect(
screen.queryByTestId('filter-distribution-Test Filter-zebra'),
).not.toBeInTheDocument();
expect(
screen.getByTestId('filter-distribution-Test Filter-apple'),
).toHaveTextContent('~60%');
});

it('should handle excluded items', () => {
Expand Down
25 changes: 25 additions & 0 deletions packages/common-utils/src/__tests__/metadata.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1768,6 +1768,31 @@ describe('Metadata', () => {
);
});

it('should key numeric and boolean values by their string form', async () => {
(mockClickhouseClient.query as jest.Mock).mockResolvedValue({
json: () =>
Promise.resolve({
data: [
{ __hdx_value: 200, __hdx_percentage: 90 },
{ __hdx_value: false, __hdx_percentage: 10 },
],
}),
});

const result = await metadata.getValuesDistribution({
chartConfig: mockChartConfig,
key: 'status',
source,
});

expect(result).toEqual(
new Map([
['200', 90],
['false', 10],
]),
);
});

it('should include alias CTEs when provided in the config', async () => {
const configWithAliases = {
...mockChartConfig,
Expand Down
5 changes: 3 additions & 2 deletions packages/common-utils/src/core/metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2404,14 +2404,15 @@ export class Metadata {
})
.then(res =>
res.json<{
__hdx_value: string;
__hdx_value: string | number | boolean;
__hdx_percentage: string | number;
}>(),
);

return new Map(
// ClickHouse JSON returns numbers and booleans unquoted
json.data.map(({ __hdx_value, __hdx_percentage }) => [
__hdx_value,
String(__hdx_value),
Number(__hdx_percentage),
]),
);
Expand Down
Loading