immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-06 01:10:00 +03:00
Code Issues Packages Projects Releases 16 Wiki Activity

fix(server): cjk migration (#24320)

Browse Source 
* join string

* use pagination instead
This commit is contained in:
Mert
2025-12-01 16:41:19 -05:00
committed by GitHub
parent 95c29a8aea
commit 7c19b0591f
1 changed files with 21 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
server/src/schema/migrations/1764483051488-OCRBigramsForCJK.ts
View File

@@ -3,21 +3,28 @@ import { tokenizeForSearch } from 'src/utils/database';
export async function up(db: Kysely<any>): Promise<void> {
await sql`truncate ${sql.table('ocr_search')}`.execute(db);
const batch = [];
for await (const { assetId, text } of db
.selectFrom('asset_ocr')
.select(['assetId', sql<string>`string_agg(text, ' ')`.as('text')])
.groupBy('assetId')
.stream()) {
batch.push({ assetId, text: tokenizeForSearch(text) });
if (batch.length >= 5000) {
await db.insertInto('ocr_search').values(batch).execute();
batch.length = 0;
}
}
if (batch.length > 0) {
await db.insertInto('ocr_search').values(batch).execute();
let lastAssetId: string | undefined;
while (true) {
const rows = await db
.selectFrom('asset_ocr')
.select(['assetId', sql<string>`string_agg(text, ' ')`.as('text')])
.$if(lastAssetId !== undefined, (qb) => qb.where('assetId', '>', lastAssetId))
.groupBy('assetId')
.orderBy('assetId')
.limit(5000)
.execute();
if (rows.length === 0) {
break;
}
await db
.insertInto('ocr_search')
.values(rows.map(({ assetId, text }) => ({ assetId, text: tokenizeForSearch(text).join(' ') })))
.execute();
lastAssetId = rows.at(-1)!.assetId;
}
}