Files
immich/server/src/infra/infra.utils.ts
Michael Manganiello e262298090 fix(server): Split database queries based on PostgreSQL bound params limit (#6034)
* fix(server): Split database queries based on PostgreSQL bound params limit

PostgreSQL uses a 16-bit integer to indicate the number of bound
parameters.

This means that the maximum number of parameters for any query is 65535.
Any query that tries to bind more than that (e.g. searching by a list of
IDs) requires splitting the query into multiple chunks.

This change includes refactoring every Repository that runs queries
using a list of ids, and either flattening or merging results.

Fixes #5788, #5997.

Also, potentially a fix for #4648 (at least based on
[this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)).

References:

* https://github.com/typeorm/typeorm/issues/7565
* [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND)

* misc: Create Chunked decorator to simplify implementation

* feat: Add ChunkedArray/ChunkedSet decorators
2024-01-06 20:36:12 -05:00

85 lines
3.2 KiB
TypeScript

import { Paginated, PaginationOptions } from '@app/domain';
import _ from 'lodash';
import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
import { chunks, setUnion } from '../domain/domain.util';
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
* or LessThanOrEqual when only one parameter is specified.
*/
export function OptionalBetween<T>(from?: T, to?: T) {
if (from && to) {
return Between(from, to);
} else if (from) {
return MoreThanOrEqual(from);
} else if (to) {
return LessThanOrEqual(to);
}
}
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
paginationOptions: PaginationOptions,
searchOptions?: FindOneOptions<Entity>,
): Paginated<Entity> {
const items = await repository.find({
...searchOptions,
// Take one more item to check if there's a next page
take: paginationOptions.take + 1,
skip: paginationOptions.skip,
});
const hasNextPage = items.length > paginationOptions.take;
items.splice(paginationOptions.take);
return { items, hasNextPage };
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
/**
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
* to overcome the maximum number of parameters allowed by the database driver.
*
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
* @param options.flatten Whether to flatten the results. Defaults to false.
*/
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
const originalMethod = descriptor.value;
const paramIndex = options.paramIndex ?? 0;
descriptor.value = async function (...args: any[]) {
const arg = args[paramIndex];
// Early return if argument length is less than or equal to the chunk size.
if (
(arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
) {
return await originalMethod.apply(this, args);
}
return Promise.all(
chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
}),
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
};
};
}
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: _.flatten });
}
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}