feat: ocr (#18836)

* feat: add OCR functionality and related configurations

* chore: update labeler configuration for machine learning files

* feat(i18n): enhance OCR model descriptions and add orientation classification and unwarping features

* chore: update Dockerfile to include ccache for improved build performance

* feat(ocr): enhance OCR model configuration with orientation classification and unwarping options, update PaddleOCR integration, and improve response structure

* refactor(ocr): remove OCR_CLEANUP job from enum and type definitions

* refactor(ocr): remove obsolete OCR entity and migration files, and update asset job status and schema to accommodate new OCR table structure

* refactor(ocr): update OCR schema and response structure to use individual coordinates instead of bounding box, and adjust related service and repository files

* feat: enhance OCR configuration and functionality

- Updated OCR settings to include minimum detection box score, minimum detection score, and minimum recognition score.
- Refactored PaddleOCRecognizer to utilize new scoring parameters.
- Introduced new database tables for asset OCR data and search functionality.
- Modified related services and repositories to support the new OCR features.
- Updated translations for improved clarity in settings UI.

* sql changes

* use rapidocr

* change dto

* update web

* update lock

* update api

* store positions as normalized floats

* match column order in db

* update admin ui settings descriptions

fix max resolution key

set min threshold to 0.1

fix bind

* apply config correctly, adjust defaults

* unnecessary model type

* unnecessary sources

* fix(ocr): switch RapidOCR lang type from LangDet to LangRec

* fix(ocr): expose lang_type (LangRec.CH) and font_path on OcrOptions for RapidOCR

* fix(ocr): make OCR text search case- and accent-insensitive using ILIKE + unaccent

* fix(ocr): add OCR search fields

* fix: Add OCR database migration and update ML prediction logic.

* trigrams are already case insensitive

* add tests

* format

* update migrations

* wrong uuid function

* linting

* maybe fix medium tests

* formatting

* fix weblate check

* openapi

* sql

* minor fixes

* maybe fix medium tests part 2

* passing medium tests

* format web

* readd sql

* format dart

* disabled in e2e

* chore: translation ordering

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/schema/index.ts

@@ -35,6 +35,7 @@ import { AssetFileTable } from 'src/schema/tables/asset-file.table';
 import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table';
 import { AssetMetadataAuditTable } from 'src/schema/tables/asset-metadata-audit.table';
 import { AssetMetadataTable } from 'src/schema/tables/asset-metadata.table';
+import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
 import { AssetTable } from 'src/schema/tables/asset.table';
 import { AuditTable } from 'src/schema/tables/audit.table';
 import { FaceSearchTable } from 'src/schema/tables/face-search.table';
@@ -47,6 +48,7 @@ import { MemoryTable } from 'src/schema/tables/memory.table';
 import { MoveTable } from 'src/schema/tables/move.table';
 import { NaturalEarthCountriesTable } from 'src/schema/tables/natural-earth-countries.table';
 import { NotificationTable } from 'src/schema/tables/notification.table';
+import { OcrSearchTable } from 'src/schema/tables/ocr-search.table';
 import { PartnerAuditTable } from 'src/schema/tables/partner-audit.table';
 import { PartnerTable } from 'src/schema/tables/partner.table';
 import { PersonAuditTable } from 'src/schema/tables/person-audit.table';
@@ -87,6 +89,7 @@ export class ImmichDatabase {
     AssetMetadataTable,
     AssetMetadataAuditTable,
     AssetJobStatusTable,
+    AssetOcrTable,
     AssetTable,
     AssetFileTable,
     AuditTable,
@@ -101,6 +104,7 @@ export class ImmichDatabase {
     MoveTable,
     NaturalEarthCountriesTable,
     NotificationTable,
+    OcrSearchTable,
     PartnerAuditTable,
     PartnerTable,
     PersonTable,
@@ -174,6 +178,8 @@ export interface DB {
   asset_metadata: AssetMetadataTable;
   asset_metadata_audit: AssetMetadataAuditTable;
   asset_job_status: AssetJobStatusTable;
+  asset_ocr: AssetOcrTable;
+  ocr_search: OcrSearchTable;
 
   audit: AuditTable;
 

server/src/schema/migrations/1758705774125-CreateAssetOCRTable.ts

@@ -0,0 +1,16 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TABLE "asset_ocr" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "assetId" uuid NOT NULL, "x1" real NOT NULL, "y1" real NOT NULL, "x2" real NOT NULL, "y2" real NOT NULL, "x3" real NOT NULL, "y3" real NOT NULL, "x4" real NOT NULL, "y4" real NOT NULL, "boxScore" real NOT NULL, "textScore" real NOT NULL, "text" text NOT NULL);`.execute(
+    db,
+  );
+  await sql`ALTER TABLE "asset_ocr" ADD CONSTRAINT "asset_ocr_pkey" PRIMARY KEY ("id");`.execute(db);
+  await sql`ALTER TABLE "asset_ocr" ADD CONSTRAINT "asset_ocr_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(
+    db,
+  );
+  await sql`CREATE INDEX "asset_ocr_assetId_idx" ON "asset_ocr" ("assetId")`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TABLE "asset_ocr";`.execute(db);
+}

server/src/schema/migrations/1758705789125-CreateOCRSearchTable.ts

@@ -0,0 +1,20 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TABLE "ocr_search" ("assetId" uuid NOT NULL, "text" text NOT NULL);`.execute(db);
+  await sql`ALTER TABLE "ocr_search" ADD CONSTRAINT "ocr_search_pkey" PRIMARY KEY ("assetId");`.execute(db);
+  await sql`ALTER TABLE "ocr_search" ADD CONSTRAINT "ocr_search_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(
+    db,
+  );
+  await sql`CREATE INDEX "idx_ocr_search_text" ON "ocr_search" USING gin (f_unaccent("text") gin_trgm_ops);`.execute(
+    db,
+  );
+  await sql`INSERT INTO "migration_overrides" ("name", "value") VALUES ('index_idx_ocr_search_text', '{"type":"index","name":"idx_ocr_search_text","sql":"CREATE INDEX \\"idx_ocr_search_text\\" ON \\"ocr_search\\" USING gin (f_unaccent(\\"text\\") gin_trgm_ops);"}'::jsonb);`.execute(
+    db,
+  );
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TABLE "ocr_search";`.execute(db);
+  await sql`DELETE FROM "migration_overrides" WHERE "name" = 'index_idx_ocr_search_text';`.execute(db);
+}

server/src/schema/migrations/1758705804128-UpsertOcrAssetJobStatus.ts

@@ -0,0 +1,9 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset_job_status" ADD "ocrAt" timestamp with time zone;`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset_job_status" DROP COLUMN "ocrAt";`.execute(db);
+}

server/src/schema/tables/asset-job-status.table.ts

@@ -20,4 +20,7 @@ export class AssetJobStatusTable {
 
   @Column({ type: 'timestamp with time zone', nullable: true })
   thumbnailAt!: Timestamp | null;
+
+  @Column({ type: 'timestamp with time zone', nullable: true })
+  ocrAt!: Timestamp | null;
 }

server/src/schema/tables/asset-ocr.table.ts

@@ -0,0 +1,45 @@
+import { AssetTable } from 'src/schema/tables/asset.table';
+import { Column, ForeignKeyColumn, Generated, PrimaryGeneratedColumn, Table } from 'src/sql-tools';
+
+@Table('asset_ocr')
+export class AssetOcrTable {
+  @PrimaryGeneratedColumn()
+  id!: Generated<string>;
+
+  @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
+  assetId!: string;
+
+  // box positions are normalized, with values between 0 and 1
+  @Column({ type: 'real' })
+  x1!: number;
+
+  @Column({ type: 'real' })
+  y1!: number;
+
+  @Column({ type: 'real' })
+  x2!: number;
+
+  @Column({ type: 'real' })
+  y2!: number;
+
+  @Column({ type: 'real' })
+  x3!: number;
+
+  @Column({ type: 'real' })
+  y3!: number;
+
+  @Column({ type: 'real' })
+  x4!: number;
+
+  @Column({ type: 'real' })
+  y4!: number;
+
+  @Column({ type: 'real' })
+  boxScore!: number;
+
+  @Column({ type: 'real' })
+  textScore!: number;
+
+  @Column({ type: 'text' })
+  text!: string;
+}

server/src/schema/tables/ocr-search.table.ts

@@ -0,0 +1,20 @@
+import { AssetTable } from 'src/schema/tables/asset.table';
+import { Column, ForeignKeyColumn, Index, Table } from 'src/sql-tools';
+
+@Table('ocr_search')
+@Index({
+  name: 'idx_ocr_search_text',
+  using: 'gin',
+  expression: 'f_unaccent("text") gin_trgm_ops',
+})
+export class OcrSearchTable {
+  @ForeignKeyColumn(() => AssetTable, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE',
+    primary: true,
+  })
+  assetId!: string;
+
+  @Column({ type: 'text' })
+  text!: string;
+}