1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-06 06:15:22 +02:00

fix(app/edge-jobs): edge job results page crash at scale (#954)

This commit is contained in:
LP B 2025-08-04 17:10:46 +02:00 committed by GitHub
parent d306d7a983
commit a472de1919
27 changed files with 2595 additions and 107 deletions

View file

@ -0,0 +1,117 @@
import {
queryOptionsFromTableState,
queryParamsFromQueryOptions,
} from './listQueryParams';
import {
withPaginationHeaders,
withPaginationQueryParams,
} from './pagination.types';
import {
makeIsSortTypeFunc,
sortOptionsFromColumns,
withSortQuery,
} from './sort.types';
const sortOptions = sortOptionsFromColumns([
{ enableSorting: true },
{ id: 'one' },
{ id: 'two', enableSorting: true },
{ accessorKey: 'three', enableSorting: true },
{ id: 'four', enableSorting: true, accessorKey: 'four_key' },
]);
describe('listQueryParams', () => {
test('queryOptionsFromTableState', () => {
const fns = {
setPageSize: () => {},
setSearch: () => {},
setSortBy: () => {},
};
expect(
queryOptionsFromTableState(
{
page: 5,
pageSize: 10,
search: 'something',
sortBy: { id: 'one', desc: false },
...fns,
},
sortOptions
)
).toStrictEqual({
search: 'something',
sort: 'one',
order: 'asc',
page: 5,
pageLimit: 10,
});
});
test('queryParamsFromQueryOptions', () => {
expect(
queryParamsFromQueryOptions({
search: 'something',
page: 5,
pageLimit: 10,
sort: 'one',
order: 'asc',
})
).toStrictEqual({
search: 'something',
sort: 'one',
order: 'asc',
start: 50,
limit: 10,
});
});
});
describe('pagination.types', () => {
test('withPaginationQueryParams', () => {
expect(withPaginationQueryParams({ page: 5, pageLimit: 10 })).toStrictEqual(
{
start: 50,
limit: 10,
}
);
});
test('withPaginationHeaders', () => {
expect(
withPaginationHeaders({
data: [],
headers: { 'x-total-count': 10, 'x-total-available': 100 },
})
).toStrictEqual({
data: [],
totalCount: 10,
totalAvailable: 100,
});
});
});
describe('sort.types', () => {
test('makeIsSortType', () => {
const isSortType = makeIsSortTypeFunc(sortOptions);
expect(typeof isSortType).toBe('function');
expect(isSortType('one')).toBe(true);
expect(isSortType('something_else')).toBe(false);
});
test('withSortQuery', () => {
expect(
withSortQuery({ id: 'one', desc: false }, sortOptions)
).toStrictEqual({ sort: 'one', order: 'asc' });
expect(
withSortQuery({ id: 'three', desc: true }, sortOptions)
).toStrictEqual({ sort: 'three', order: 'desc' });
expect(
withSortQuery({ id: 'something_else', desc: true }, sortOptions)
).toStrictEqual({ sort: undefined, order: 'desc' });
});
test('sortOptionsFromColumns', () => {
expect(sortOptions).toEqual(['one', 'two', 'three', 'four']);
});
});

View file

@ -0,0 +1,65 @@
import { BasicTableSettings } from '@@/datatables/types';
import { TableState } from '@@/datatables/useTableState';
import {
PaginationQuery,
PaginationQueryParams,
withPaginationQueryParams,
} from './pagination.types';
import { SearchQuery, SearchQueryParams } from './search.types';
import {
SortOptions,
SortQuery,
SortQueryParams,
withSortQuery,
} from './sort.types';
export type BaseQueryOptions<T extends SortOptions> = SearchQuery &
SortQuery<T> &
PaginationQuery;
/**
* Utility function to transform a TableState (base form) to a query options object
* Used to unify backend pagination common cases
*
* @param tableState TableState {search, sortBy: {id:string, desc:bool }, page, pageSize}
* @param sortOptions SortOptions (generated from columns)
* @returns BaseQuery {search, sort, order, page, pageLimit}
*/
export function queryOptionsFromTableState<T extends SortOptions>(
tableState: TableState<BasicTableSettings> & { page: number },
sortOptions: T
): BaseQueryOptions<T> {
return {
// search/filter
search: tableState.search,
// sorting
...withSortQuery(tableState.sortBy, sortOptions),
// pagination
page: tableState.page,
pageLimit: tableState.pageSize,
};
}
export type BaseQueryParams<T extends SortOptions> = SearchQueryParams &
SortQueryParams<T> &
PaginationQueryParams;
/**
*
* @param query BaseQueryOptions
* @returns BaseQueryParams {search, sort, order, start, limit}
*/
export function queryParamsFromQueryOptions<T extends SortOptions>(
query: BaseQueryOptions<T>
): BaseQueryParams<T> {
return {
// search/filter
search: query.search,
// sorting
sort: query.sort,
order: query.order,
// paginattion
...withPaginationQueryParams(query),
};
}

View file

@ -0,0 +1,114 @@
import { AxiosResponse } from 'axios';
/**
* Used to define axios query functions parameters for queries that support backend pagination
*
* **Example**
*
* ```ts
* type QueryParams = PaginationQueryParams;
*
* async function getSomething({ start, limit }: QueryParams = {}) {
* try {
* const { data } = await axios.get<APIType>(
* buildUrl(),
* { params: { start, limit } },
* );
* return data;
* } catch (err) {
* throw parseAxiosError(err as Error, 'Unable to retrieve something');
* }
* }
*```
*/
export type PaginationQueryParams = {
start?: number;
limit?: number;
};
/**
* Used to define react-query query functions parameters for queries that support backend pagination
*
* Example:
*
* ```ts
* type Query = PaginationQuery;
*
* function useSomething({
* page = 0,
* pageLimit = 10,
* ...query
* }: Query = {}) {
* return useQuery(
* [ ...queryKeys.base(), { page, pageLimit, ...query } ],
* async () => {
* const start = (page - 1) * pageLimit + 1;
* return getSomething({ start, limit: pageLimit, ...query });
* },
* {
* ...withError('Failure retrieving something'),
* }
* );
* }
* ```
*/
export type PaginationQuery = {
page?: number;
pageLimit?: number;
};
/**
* Utility function to convert PaginationQuery to PaginationQueryParams
*
* **Example**
*
* ```ts
* function getSomething(params: PaginationQueryParams) {...}
*
* function useSomething(query: PaginationQuery) {
* return useQuery(
* [ ...queryKeys.base(), query ],
* async () => getSomething({ ...query, ...withPaginationQueryParams(query) })
* )
* }
* ```
*/
export function withPaginationQueryParams({
page = 0,
pageLimit = 10,
}: PaginationQuery): PaginationQueryParams {
const start = page * pageLimit;
return {
start,
limit: pageLimit,
};
}
export type PaginatedResults<T> = {
data: T;
totalCount: number;
totalAvailable: number;
};
/**
* Utility function to extract total count from AxiosResponse headers
*
* @param param0 AxiosReponse-like object {data, headers}
* @returns PaginatedResults {data, totalCount, totalAvailable}
*/
export function withPaginationHeaders<T = unknown>({
data,
headers,
}: {
data: AxiosResponse<T>['data'];
headers: AxiosResponse<T>['headers'];
}): PaginatedResults<T> {
const totalCount = parseInt(headers['x-total-count'], 10);
const totalAvailable = parseInt(headers['x-total-available'], 10);
return {
data,
totalCount,
totalAvailable,
};
}

View file

@ -0,0 +1,47 @@
/**
* Used to define axios query functions parameters for queries that support backend filtering by search
*
* **Example**
*
* ```ts
* type QueryParams = SearchQueryParams;
*
* async function getSomething({ search }: QueryParams = {}) {
* try {
* const { data } = await axios.get<APIType>(
* buildUrl(),
* { params: { search } },
* );
* return data;
* } catch (err) {
* throw parseAxiosError(err as Error, 'Unable to retrieve something');
* }
* }
*```
*/
export type SearchQueryParams = {
search?: string;
};
/**
* Used to define react-query query functions parameters for queries that support backend filtering by search
*
* Example:
*
* ```ts
* type Query = SearchQuery;
*
* function useSomething({ search, ...query }: Query = {}) {
* return useQuery(
* [ ...queryKeys.base(), { search, ...query } ],
* async () => getSomething({ search, ...query }),
* {
* ...withError('Failure retrieving something'),
* }
* );
* }
* ```
*/
export type SearchQuery = {
search?: string;
};

View file

@ -0,0 +1,139 @@
import { compact } from 'lodash';
import { SortableTableSettings } from '@@/datatables/types';
export type SortOptions = readonly string[];
export type SortType<T extends SortOptions> = T[number];
/**
* Used to generate the validation function that allows to check if the sort key is supported or not
*
* **Example**
*
* ```ts
* export const sortOptions: SortOptions = ['Id', 'Name'] as const;
* export const isSortType = makeIsSortTypeFunc(sortOptions)
* ```
*
* **Usage**
*
* ```ts
* // react-query hook definition
* export function useSomething({ sort, order }: SortQuery<typeof sortOptions>) { ... }
*
* // component using the react-query hook, validating the parameters used by the query
* function MyComponent() {
* const tableState = useTableState(settingsStore, tableKey);
* const { data } = useSomething(
* {
* sort: isSortType(tableState.sortBy.id) ? tableState.sortBy.id : undefined,
* order: tableState.sortBy.desc ? 'desc' : 'asc',
* },
* );
* ...
* }
* ```
*
* @param sortOptions list of supported keys
* @returns validation function
*/
export function makeIsSortTypeFunc<T extends SortOptions>(sortOptions: T) {
return (value?: string): value is SortType<T> =>
sortOptions.includes(value as SortType<T>);
}
/**
* Used to define axios query functions parameters for queries that support backend sorting
*
* **Example**
*
* ```ts
* const sortOptions: SortOptions = ['Id', 'Name'] as const; // or generated with `sortOptionsFromColumns`
* type QueryParams = SortQueryParams<typeof sortOptions>;
*
* async function getSomething({ sort, order = 'asc' }: QueryParams = {}) {
* try {
* const { data } = await axios.get<APIType>(
* buildUrl(),
* { params: { sort, order } },
* );
* return data;
* } catch (err) {
* throw parseAxiosError(err as Error, 'Unable to retrieve something');
* }
* }
*```
*/
export type SortQueryParams<T extends SortOptions> = {
sort?: SortType<T>;
order?: 'asc' | 'desc';
};
/**
* Used to define react-query query functions parameters for queries that support backend sorting
*
* Example:
*
* ```ts
* const sortOptions: SortOptions = ['Id', 'Name'] as const;
* type Query = SortQuery<typeof sortOptions>;
*
* function useSomething({
* sort,
* order = 'asc',
* ...query
* }: Query = {}) {
* return useQuery(
* [ ...queryKeys.base(), { ...query, sort, order } ],
* async () => getSomething({ ...query, sort, order }),
* {
* ...withError('Failure retrieving something'),
* }
* );
* }
* ```
*/
export type SortQuery<T extends SortOptions> = {
sort?: SortType<T>;
order?: 'asc' | 'desc';
};
/**
* Utility function to convert react-table `sortBy` state to `SortQuery` query parameter
*
* @param sortBy tableState.sortBy
* @param sortOptions SortOptions - either defined manually, or generated with `sortOptionsFromColumns`
* @returns SortQuery - object usable by react-query functions that have params extending SortQuery
*/
export function withSortQuery<T extends SortOptions>(
sortBy: SortableTableSettings['sortBy'],
sortOptions: T
): SortQuery<T> {
if (!sortBy) {
return {
sort: undefined,
order: 'asc',
};
}
const isSortType = makeIsSortTypeFunc(sortOptions);
return {
sort: isSortType(sortBy.id) ? sortBy.id : undefined,
order: sortBy.desc ? 'desc' : 'asc',
};
}
/**
* Utility function to generate SortOptions from columns definitions
* @param columns Column-like objects { id?:string; enableSorting?:boolean } to extract SortOptions from
* @returns SortOptions
*/
export function sortOptionsFromColumns(
columns: { id?: string; enableSorting?: boolean; accessorKey?: string }[]
): SortOptions {
return compact(
columns.map((c) =>
c.enableSorting === false ? undefined : c.id ?? c.accessorKey
)
);
}

View file

@ -70,6 +70,7 @@ export interface Props<D extends DefaultType> extends AutomationTestingProps {
getRowCanExpand?(row: Row<D>): boolean;
noWidget?: boolean;
extendTableOptions?: (options: TableOptions<D>) => TableOptions<D>;
onSearchChange?: (search: string) => void;
includeSearch?: boolean;
ariaLabel?: string;
id?: string;
@ -97,6 +98,7 @@ export function Datatable<D extends DefaultType>({
getRowCanExpand,
'data-cy': dataCy,
onPageChange = () => {},
onSearchChange = () => {},
page,
totalCount = dataset.length,
isServerSidePagination = false,
@ -158,7 +160,12 @@ export function Datatable<D extends DefaultType>({
getRowCanExpand,
getColumnCanGlobalFilter,
...(isServerSidePagination
? { manualPagination: true, pageCount }
? {
pageCount,
manualPagination: true,
manualFiltering: true,
manualSorting: true,
}
: {
getSortedRowModel: getSortedRowModel(),
}),
@ -231,6 +238,7 @@ export function Datatable<D extends DefaultType>({
function handleSearchBarChange(search: string) {
tableInstance.setGlobalFilter({ search });
settings.setSearch(search);
onSearchChange(search);
}
function handlePageChange(page: number) {

View file

@ -6,16 +6,18 @@ import { keyBuilder } from '@/react/hooks/useLocalStorage';
export type DefaultType = object;
export interface PaginationTableSettings {
pageSize: number;
setPageSize: (pageSize: number) => void;
}
export type ZustandSetFunc<T> = (
partial: T | Partial<T> | ((state: T) => T | Partial<T>),
replace?: boolean | undefined
) => void;
// pagination (page size dropdown)
// for both backend and frontend paginations
export interface PaginationTableSettings {
pageSize: number;
setPageSize: (pageSize: number) => void;
}
export function paginationSettings<T extends PaginationTableSettings>(
set: ZustandSetFunc<T>
): PaginationTableSettings {
@ -25,6 +27,24 @@ export function paginationSettings<T extends PaginationTableSettings>(
};
}
// pagination (page number selector)
// for backend pagination
export interface BackendPaginationTableSettings {
page: number;
setPage: (page: number) => void;
}
export function backendPaginationSettings<
T extends BackendPaginationTableSettings,
>(set: ZustandSetFunc<T>): BackendPaginationTableSettings {
return {
page: 0,
setPage: (page: number) => set((s) => ({ ...s, page })),
};
}
// sorting
// arrows in datatable column headers
export interface SortableTableSettings {
sortBy: { id: string; desc: boolean } | undefined;
setSortBy: (id: string | undefined, desc: boolean) => void;
@ -47,6 +67,8 @@ export function sortableSettings<T extends SortableTableSettings>(
};
}
// hidding columns
// datatable options allowing to hide columns
export interface SettableColumnsTableSettings {
hiddenColumns: string[];
setHiddenColumns: (hiddenColumns: string[]) => void;
@ -63,6 +85,7 @@ export function hiddenColumnsSettings<T extends SettableColumnsTableSettings>(
};
}
// auto refresh settings
export interface RefreshableTableSettings {
autoRefreshRate: number;
setAutoRefreshRate: (autoRefreshRate: number) => void;
@ -70,7 +93,7 @@ export interface RefreshableTableSettings {
export function refreshableSettings<T extends RefreshableTableSettings>(
set: ZustandSetFunc<T>,
autoRefreshRate = 0
autoRefreshRate: number = 0
): RefreshableTableSettings {
return {
autoRefreshRate,

View file

@ -1,10 +0,0 @@
diff a/app/react/components/form-components/Input/Select.tsx b/app/react/components/form-components/Input/Select.tsx (rejected hunks)
@@ -10,7 +10,7 @@ export interface Option<T extends string | number>
disabled?: boolean;
}
-interface Props<T extends string | number> {
+interface Props<T extends string | number> extends AutomationTestingProps {
options: Option<T>[];
}

View file

@ -1,18 +1,16 @@
import { List } from 'lucide-react';
import { useMemo } from 'react';
import { Environment } from '@/react/portainer/environments/types';
import { useEnvironmentList } from '@/react/portainer/environments/queries';
import { queryOptionsFromTableState } from '@/react/common/api/listQueryParams';
import { Datatable } from '@@/datatables';
import { useTableState } from '@@/datatables/useTableState';
import { withMeta } from '@@/datatables/extend-options/withMeta';
import { mergeOptions } from '@@/datatables/extend-options/mergeOptions';
import { EdgeJob, JobResult, LogsStatus } from '../../types';
import { EdgeJob, LogsStatus } from '../../types';
import { useJobResults } from '../../queries/jobResults/useJobResults';
import { columns } from './columns';
import { columns, sortOptions } from './columns';
import { createStore } from './datatable-store';
const tableKey = 'edge-job-results';
@ -22,8 +20,9 @@ export function ResultsDatatable({ jobId }: { jobId: EdgeJob['Id'] }) {
const tableState = useTableState(store, tableKey);
const jobResultsQuery = useJobResults(jobId, {
...queryOptionsFromTableState({ ...tableState }, sortOptions),
refetchInterval(dataset) {
const anyCollecting = dataset?.some(
const anyCollecting = dataset?.data.some(
(r) => r.LogsStatus === LogsStatus.Pending
);
@ -35,40 +34,17 @@ export function ResultsDatatable({ jobId }: { jobId: EdgeJob['Id'] }) {
},
});
const environmentIds = jobResultsQuery.data?.map(
(result) => result.EndpointId
);
const environmentsQuery = useEnvironmentList(
{ endpointIds: environmentIds },
{ enabled: !!environmentIds && !jobResultsQuery.isLoading }
);
const dataset = useMemo(
() =>
jobResultsQuery.isLoading || environmentsQuery.isLoading
? []
: associateEndpointsToResults(
jobResultsQuery.data || [],
environmentsQuery.environments
),
[
environmentsQuery.environments,
environmentsQuery.isLoading,
jobResultsQuery.data,
jobResultsQuery.isLoading,
]
);
const dataset = jobResultsQuery.data?.data || [];
return (
<Datatable
disableSelect
columns={columns}
dataset={dataset}
isLoading={jobResultsQuery.isLoading || environmentsQuery.isLoading}
title="Results"
titleIcon={List}
columns={columns}
disableSelect
dataset={dataset}
settingsManager={tableState}
isLoading={jobResultsQuery.isLoading}
extendTableOptions={mergeOptions(
withMeta({
table: 'edge-job-results',
@ -76,21 +52,11 @@ export function ResultsDatatable({ jobId }: { jobId: EdgeJob['Id'] }) {
})
)}
data-cy="edge-job-results-datatable"
isServerSidePagination
page={tableState.page}
onPageChange={tableState.setPage}
onSearchChange={() => tableState.setPage(0)}
totalCount={jobResultsQuery.data?.totalCount || 0}
/>
);
}
function associateEndpointsToResults(
results: Array<JobResult>,
environments: Array<Environment>
) {
return results.map((result) => {
const environment = environments.find(
(environment) => environment.Id === result.EndpointId
);
return {
...result,
Endpoint: environment,
};
});
}

View file

@ -1,18 +1,20 @@
import { CellContext, createColumnHelper } from '@tanstack/react-table';
import { sortOptionsFromColumns } from '@/react/common/api/sort.types';
import { Button } from '@@/buttons';
import { LogsStatus } from '../../types';
import { JobResult, LogsStatus } from '../../types';
import { useDownloadLogsMutation } from '../../queries/jobResults/useDownloadLogsMutation';
import { useClearLogsMutation } from '../../queries/jobResults/useClearLogsMutation';
import { useCollectLogsMutation } from '../../queries/jobResults/useCollectLogsMutation';
import { DecoratedJobResult, getTableMeta } from './types';
import { getTableMeta } from './types';
const columnHelper = createColumnHelper<DecoratedJobResult>();
const columnHelper = createColumnHelper<JobResult>();
export const columns = [
columnHelper.accessor('Endpoint.Name', {
columnHelper.accessor('EndpointName', {
header: 'Environment',
meta: {
className: 'w-1/2',
@ -30,7 +32,7 @@ export const columns = [
function ActionsCell({
row: { original: item },
table,
}: CellContext<DecoratedJobResult, unknown>) {
}: CellContext<JobResult, unknown>) {
const tableMeta = getTableMeta(table.options.meta);
const id = tableMeta.jobId;
@ -51,13 +53,13 @@ function ActionsCell({
<>
<Button
onClick={() => downloadLogsMutation.mutate(item.EndpointId)}
data-cy={`edge-job-download-logs-${item.Endpoint?.Name}`}
data-cy={`edge-job-download-logs-${item.EndpointName}`}
>
Download logs
</Button>
<Button
onClick={() => clearLogsMutations.mutate(item.EndpointId)}
data-cy={`edge-job-clear-logs-${item.Endpoint?.Name}`}
data-cy={`edge-job-clear-logs-${item.EndpointName}`}
>
Clear logs
</Button>
@ -68,10 +70,12 @@ function ActionsCell({
return (
<Button
onClick={() => collectLogsMutation.mutate(item.EndpointId)}
data-cy={`edge-job-retrieve-logs-${item.Endpoint?.Name}`}
data-cy={`edge-job-retrieve-logs-${item.EndpointName}`}
>
Retrieve logs
</Button>
);
}
}
export const sortOptions = sortOptionsFromColumns(columns);

View file

@ -3,12 +3,18 @@ import {
createPersistedStore,
BasicTableSettings,
RefreshableTableSettings,
BackendPaginationTableSettings,
backendPaginationSettings,
} from '@@/datatables/types';
interface TableSettings extends BasicTableSettings, RefreshableTableSettings {}
interface TableSettings
extends BasicTableSettings,
RefreshableTableSettings,
BackendPaginationTableSettings {}
export function createStore(storageKey: string) {
return createPersistedStore<TableSettings>(storageKey, undefined, (set) => ({
...refreshableSettings(set),
...backendPaginationSettings(set),
}));
}

View file

@ -1,10 +1,4 @@
import { Environment } from '@/react/portainer/environments/types';
import { EdgeJob, JobResult } from '../../types';
export interface DecoratedJobResult extends JobResult {
Endpoint?: Environment;
}
import { EdgeJob } from '../../types';
interface TableMeta {
table: 'edge-job-results';

View file

@ -1,35 +1,54 @@
import { useQuery } from '@tanstack/react-query';
import axios, { parseAxiosError } from '@/portainer/services/axios';
import {
PaginatedResults,
withPaginationHeaders,
} from '@/react/common/api/pagination.types';
import {
BaseQueryOptions,
BaseQueryParams,
queryParamsFromQueryOptions,
} from '@/react/common/api/listQueryParams';
import { EdgeJob, JobResult } from '../../types';
import { sortOptions } from '../../ItemView/ResultsDatatable/columns';
import { queryKeys } from './query-keys';
import { buildUrl } from './build-url';
type QueryOptions = BaseQueryOptions<typeof sortOptions>;
type RefetchInterval =
| number
| false
| ((data: PaginatedResults<Array<JobResult>> | undefined) => number | false);
export function useJobResults(
id: EdgeJob['Id'],
{
refetchInterval,
...query
}: {
refetchInterval?:
| number
| false
| ((data: Array<JobResult> | undefined) => number | false);
} = {}
refetchInterval?: RefetchInterval;
} & QueryOptions = {}
) {
return useQuery({
queryKey: queryKeys.base(id),
queryFn: () => getJobResults(id),
queryKey: [...queryKeys.base(id), query],
queryFn: () => getJobResults(id, queryParamsFromQueryOptions(query)),
refetchInterval,
});
}
async function getJobResults(id: EdgeJob['Id']) {
try {
const { data } = await axios.get<Array<JobResult>>(buildUrl({ id }));
type QueryParams = BaseQueryParams<typeof sortOptions>;
return data;
async function getJobResults(id: EdgeJob['Id'], params?: QueryParams) {
try {
const response = await axios.get<Array<JobResult>>(
`edge_jobs/${id}/tasks`,
{ params }
);
return withPaginationHeaders(response);
} catch (err) {
throw parseAxiosError(err, 'Failed fetching edge job results');
}

View file

@ -1,4 +1,7 @@
import { EnvironmentId } from '@/react/portainer/environments/types';
import {
Environment,
EnvironmentId,
} from '@/react/portainer/environments/types';
export interface EdgeJob {
Id: number;
@ -28,5 +31,6 @@ interface EndpointMeta {
export interface JobResult {
Id: string;
EndpointId: EnvironmentId;
EndpointName: Environment['Name'];
LogsStatus: LogsStatus;
}