feat: ocr (#18836)

* feat: add OCR functionality and related configurations

* chore: update labeler configuration for machine learning files

* feat(i18n): enhance OCR model descriptions and add orientation classification and unwarping features

* chore: update Dockerfile to include ccache for improved build performance

* feat(ocr): enhance OCR model configuration with orientation classification and unwarping options, update PaddleOCR integration, and improve response structure

* refactor(ocr): remove OCR_CLEANUP job from enum and type definitions

* refactor(ocr): remove obsolete OCR entity and migration files, and update asset job status and schema to accommodate new OCR table structure

* refactor(ocr): update OCR schema and response structure to use individual coordinates instead of bounding box, and adjust related service and repository files

* feat: enhance OCR configuration and functionality

- Updated OCR settings to include minimum detection box score, minimum detection score, and minimum recognition score.
- Refactored PaddleOCRecognizer to utilize new scoring parameters.
- Introduced new database tables for asset OCR data and search functionality.
- Modified related services and repositories to support the new OCR features.
- Updated translations for improved clarity in settings UI.

* sql changes

* use rapidocr

* change dto

* update web

* update lock

* update api

* store positions as normalized floats

* match column order in db

* update admin ui settings descriptions

fix max resolution key

set min threshold to 0.1

fix bind

* apply config correctly, adjust defaults

* unnecessary model type

* unnecessary sources

* fix(ocr): switch RapidOCR lang type from LangDet to LangRec

* fix(ocr): expose lang_type (LangRec.CH) and font_path on OcrOptions for RapidOCR

* fix(ocr): make OCR text search case- and accent-insensitive using ILIKE + unaccent

* fix(ocr): add OCR search fields

* fix: Add OCR database migration and update ML prediction logic.

* trigrams are already case insensitive

* add tests

* format

* update migrations

* wrong uuid function

* linting

* maybe fix medium tests

* formatting

* fix weblate check

* openapi

* sql

* minor fixes

* maybe fix medium tests part 2

* passing medium tests

* format web

* readd sql

* format dart

* disabled in e2e

* chore: translation ordering

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/config.ts

@@ -74,6 +74,13 @@ export interface SystemConfig {
       minFaces: number;
       maxDistance: number;
     };
+    ocr: {
+      enabled: boolean;
+      modelName: string;
+      minDetectionScore: number;
+      minRecognitionScore: number;
+      maxResolution: number;
+    };
   };
   map: {
     enabled: boolean;
@@ -227,6 +234,7 @@ export const defaults = Object.freeze<SystemConfig>({
     [QueueName.ThumbnailGeneration]: { concurrency: 3 },
     [QueueName.VideoConversion]: { concurrency: 1 },
     [QueueName.Notification]: { concurrency: 5 },
+    [QueueName.Ocr]: { concurrency: 1 },
   },
   logging: {
     enabled: true,
@@ -255,6 +263,13 @@ export const defaults = Object.freeze<SystemConfig>({
       maxDistance: 0.5,
       minFaces: 3,
     },
+    ocr: {
+      enabled: true,
+      modelName: 'PP-OCRv5_mobile',
+      minDetectionScore: 0.5,
+      minRecognitionScore: 0.8,
+      maxResolution: 736,
+    },
   },
   map: {
     enabled: true,

server/src/dtos/job.dto.ts

@@ -93,4 +93,7 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
 
   @ApiProperty({ type: JobStatusDto })
   [QueueName.BackupDatabase]!: JobStatusDto;
+
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.Ocr]!: JobStatusDto;
 }

server/src/dtos/model-config.dto.ts

@@ -46,3 +46,25 @@ export class FacialRecognitionConfig extends ModelConfig {
   @ApiProperty({ type: 'integer' })
   minFaces!: number;
 }
+
+export class OcrConfig extends ModelConfig {
+  @IsNumber()
+  @Min(1)
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  maxResolution!: number;
+
+  @IsNumber()
+  @Min(0.1)
+  @Max(1)
+  @Type(() => Number)
+  @ApiProperty({ type: 'number', format: 'double' })
+  minDetectionScore!: number;
+
+  @IsNumber()
+  @Min(0.1)
+  @Max(1)
+  @Type(() => Number)
+  @ApiProperty({ type: 'number', format: 'double' })
+  minRecognitionScore!: number;
+}

server/src/dtos/search.dto.ts

@@ -101,6 +101,11 @@ class BaseSearchDto {
   @Max(5)
   @Min(-1)
   rating?: number;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  ocr?: string;
 }
 
 class BaseSearchWithResultsDto extends BaseSearchDto {

server/src/dtos/server.dto.ts

@@ -171,6 +171,7 @@ export class ServerFeaturesDto {
   sidecar!: boolean;
   search!: boolean;
   email!: boolean;
+  ocr!: boolean;
 }
 
 export interface ReleaseNotification {

server/src/dtos/system-config.dto.ts

@@ -15,7 +15,7 @@ import {
   ValidateNested,
 } from 'class-validator';
 import { SystemConfig } from 'src/config';
-import { CLIPConfig, DuplicateDetectionConfig, FacialRecognitionConfig } from 'src/dtos/model-config.dto';
+import { CLIPConfig, DuplicateDetectionConfig, FacialRecognitionConfig, OcrConfig } from 'src/dtos/model-config.dto';
 import {
   AudioCodec,
   CQMode,
@@ -201,6 +201,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
   @Type(() => JobSettingsDto)
   [QueueName.FaceDetection]!: JobSettingsDto;
 
+  @ApiProperty({ type: JobSettingsDto })
+  @ValidateNested()
+  @IsObject()
+  @Type(() => JobSettingsDto)
+  [QueueName.Ocr]!: JobSettingsDto;
+
   @ApiProperty({ type: JobSettingsDto })
   @ValidateNested()
   @IsObject()
@@ -296,6 +302,11 @@ class SystemConfigMachineLearningDto {
   @ValidateNested()
   @IsObject()
   facialRecognition!: FacialRecognitionConfig;
+
+  @Type(() => OcrConfig)
+  @ValidateNested()
+  @IsObject()
+  ocr!: OcrConfig;
 }
 
 enum MapTheme {

server/src/enum.ts

@@ -513,6 +513,7 @@ export enum QueueName {
   Library = 'library',
   Notification = 'notifications',
   BackupDatabase = 'backupDatabase',
+  Ocr = 'ocr',
 }
 
 export enum JobName {
@@ -585,6 +586,10 @@ export enum JobName {
   TagCleanup = 'TagCleanup',
 
   VersionCheck = 'VersionCheck',
+
+  // OCR
+  OcrQueueAll = 'OcrQueueAll',
+  Ocr = 'Ocr',
 }
 
 export enum JobCommand {

server/src/queries/asset.job.repository.sql

@@ -285,6 +285,23 @@ from
 where
   "asset"."id" = $2
 
+-- AssetJobRepository.getForOcr
+select
+  "asset"."visibility",
+  (
+    select
+      "asset_file"."path"
+    from
+      "asset_file"
+    where
+      "asset_file"."assetId" = "asset"."id"
+      and "asset_file"."type" = $1
+  ) as "previewFile"
+from
+  "asset"
+where
+  "asset"."id" = $2
+
 -- AssetJobRepository.getForSyncAssets
 select
   "asset"."id",
@@ -483,6 +500,17 @@ where
 order by
   "asset"."fileCreatedAt" desc
 
+-- AssetJobRepository.streamForOcrJob
+select
+  "asset"."id"
+from
+  "asset"
+  inner join "asset_job_status" on "asset_job_status"."assetId" = "asset"."id"
+where
+  "asset_job_status"."ocrAt" is null
+  and "asset"."deletedAt" is null
+  and "asset"."visibility" != $1
+
 -- AssetJobRepository.streamForMigrationJob
 select
   "id"

server/src/queries/ocr.repository.sql

@@ -0,0 +1,68 @@
+-- NOTE: This file is auto generated by ./sql-generator
+
+-- OcrRepository.getById
+select
+  "asset_ocr".*
+from
+  "asset_ocr"
+where
+  "asset_ocr"."id" = $1
+
+-- OcrRepository.getByAssetId
+select
+  "asset_ocr".*
+from
+  "asset_ocr"
+where
+  "asset_ocr"."assetId" = $1
+
+-- OcrRepository.upsert
+with
+  "deleted_ocr" as (
+    delete from "asset_ocr"
+    where
+      "assetId" = $1
+  ),
+  "inserted_ocr" as (
+    insert into
+      "asset_ocr" (
+        "assetId",
+        "x1",
+        "y1",
+        "x2",
+        "y2",
+        "x3",
+        "y3",
+        "x4",
+        "y4",
+        "text",
+        "boxScore",
+        "textScore"
+      )
+    values
+      (
+        $2,
+        $3,
+        $4,
+        $5,
+        $6,
+        $7,
+        $8,
+        $9,
+        $10,
+        $11,
+        $12,
+        $13
+      )
+  ),
+  "inserted_search" as (
+    insert into
+      "ocr_search" ("assetId", "text")
+    values
+      ($14, $15)
+    on conflict ("assetId") do update
+    set
+      "text" = "excluded"."text"
+  )
+select
+  1 as "dummy"

server/src/repositories/asset-job.repository.ts

@@ -16,6 +16,7 @@ import {
   withExifInner,
   withFaces,
   withFacesAndPeople,
+  withFilePath,
   withFiles,
 } from 'src/utils/database';
 
@@ -192,6 +193,15 @@ export class AssetJobRepository {
       .executeTakeFirst();
   }
 
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getForOcr(id: string) {
+    return this.db
+      .selectFrom('asset')
+      .select((eb) => ['asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')])
+      .where('asset.id', '=', id)
+      .executeTakeFirst();
+  }
+
   @GenerateSql({ params: [[DummyValue.UUID]] })
   getForSyncAssets(ids: string[]) {
     return this.db
@@ -348,6 +358,21 @@ export class AssetJobRepository {
       .stream();
   }
 
+  @GenerateSql({ params: [], stream: true })
+  streamForOcrJob(force?: boolean) {
+    return this.db
+      .selectFrom('asset')
+      .select(['asset.id'])
+      .$if(!force, (qb) =>
+        qb
+          .innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id')
+          .where('asset_job_status.ocrAt', 'is', null),
+      )
+      .where('asset.deletedAt', 'is', null)
+      .where('asset.visibility', '!=', AssetVisibility.Hidden)
+      .stream();
+  }
+
   @GenerateSql({ params: [DummyValue.DATE], stream: true })
   streamForMigrationJob() {
     return this.db.selectFrom('asset').select(['id']).where('asset.deletedAt', 'is', null).stream();

server/src/repositories/asset.repository.ts

@@ -205,6 +205,7 @@ export class AssetRepository {
               metadataExtractedAt: eb.ref('excluded.metadataExtractedAt'),
               previewAt: eb.ref('excluded.previewAt'),
               thumbnailAt: eb.ref('excluded.thumbnailAt'),
+              ocrAt: eb.ref('excluded.ocrAt'),
             },
             values[0],
           ),

server/src/repositories/index.ts

@@ -25,6 +25,7 @@ import { MetadataRepository } from 'src/repositories/metadata.repository';
 import { MoveRepository } from 'src/repositories/move.repository';
 import { NotificationRepository } from 'src/repositories/notification.repository';
 import { OAuthRepository } from 'src/repositories/oauth.repository';
+import { OcrRepository } from 'src/repositories/ocr.repository';
 import { PartnerRepository } from 'src/repositories/partner.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { ProcessRepository } from 'src/repositories/process.repository';
@@ -74,6 +75,7 @@ export const repositories = [
   MoveRepository,
   NotificationRepository,
   OAuthRepository,
+  OcrRepository,
   PartnerRepository,
   PersonRepository,
   ProcessRepository,

server/src/repositories/machine-learning.repository.ts

@@ -15,6 +15,7 @@ export interface BoundingBox {
 export enum ModelTask {
   FACIAL_RECOGNITION = 'facial-recognition',
   SEARCH = 'clip',
+  OCR = 'ocr',
 }
 
 export enum ModelType {
@@ -23,6 +24,7 @@ export enum ModelType {
   RECOGNITION = 'recognition',
   TEXTUAL = 'textual',
   VISUAL = 'visual',
+  OCR = 'ocr',
 }
 
 export type ModelPayload = { imagePath: string } | { text: string };
@@ -30,7 +32,11 @@ export type ModelPayload = { imagePath: string } | { text: string };
 type ModelOptions = { modelName: string };
 
 export type FaceDetectionOptions = ModelOptions & { minScore: number };
-
+export type OcrOptions = ModelOptions & {
+  minDetectionScore: number;
+  minRecognitionScore: number;
+  maxResolution: number;
+};
 type VisualResponse = { imageHeight: number; imageWidth: number };
 export type ClipVisualRequest = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: ModelOptions } };
 export type ClipVisualResponse = { [ModelTask.SEARCH]: string } & VisualResponse;
@@ -38,6 +44,21 @@ export type ClipVisualResponse = { [ModelTask.SEARCH]: string } & VisualResponse
 export type ClipTextualRequest = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: ModelOptions } };
 export type ClipTextualResponse = { [ModelTask.SEARCH]: string };
 
+export type OCR = {
+  text: string[];
+  box: number[];
+  boxScore: number[];
+  textScore: number[];
+};
+
+export type OcrRequest = {
+  [ModelTask.OCR]: {
+    [ModelType.DETECTION]: ModelOptions & { options: { minScore: number; maxResolution: number } };
+    [ModelType.RECOGNITION]: ModelOptions & { options: { minScore: number } };
+  };
+};
+export type OcrResponse = { [ModelTask.OCR]: OCR } & VisualResponse;
+
 export type FacialRecognitionRequest = {
   [ModelTask.FACIAL_RECOGNITION]: {
     [ModelType.DETECTION]: ModelOptions & { options: { minScore: number } };
@@ -53,7 +74,7 @@ export interface Face {
 
 export type FacialRecognitionResponse = { [ModelTask.FACIAL_RECOGNITION]: Face[] } & VisualResponse;
 export type DetectedFaces = { faces: Face[] } & VisualResponse;
-export type MachineLearningRequest = ClipVisualRequest | ClipTextualRequest | FacialRecognitionRequest;
+export type MachineLearningRequest = ClipVisualRequest | ClipTextualRequest | FacialRecognitionRequest | OcrRequest;
 export type TextEncodingOptions = ModelOptions & { language?: string };
 
 @Injectable()
@@ -197,6 +218,17 @@ export class MachineLearningRepository {
     return response[ModelTask.SEARCH];
   }
 
+  async ocr(imagePath: string, { modelName, minDetectionScore, minRecognitionScore, maxResolution }: OcrOptions) {
+    const request = {
+      [ModelTask.OCR]: {
+        [ModelType.DETECTION]: { modelName, options: { minScore: minDetectionScore, maxResolution } },
+        [ModelType.RECOGNITION]: { modelName, options: { minScore: minRecognitionScore } },
+      },
+    };
+    const response = await this.predict<OcrResponse>({ imagePath }, request);
+    return response[ModelTask.OCR];
+  }
+
   private async getFormData(payload: ModelPayload, config: MachineLearningRequest): Promise<FormData> {
     const formData = new FormData();
     formData.append('entries', JSON.stringify(config));

server/src/repositories/ocr.repository.ts

@@ -0,0 +1,68 @@
+import { Injectable } from '@nestjs/common';
+import { Insertable, Kysely, sql } from 'kysely';
+import { InjectKysely } from 'nestjs-kysely';
+import { DummyValue, GenerateSql } from 'src/decorators';
+import { DB } from 'src/schema';
+import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
+
+@Injectable()
+export class OcrRepository {
+  constructor(@InjectKysely() private db: Kysely<DB>) {}
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getById(id: string) {
+    return this.db.selectFrom('asset_ocr').selectAll('asset_ocr').where('asset_ocr.id', '=', id).executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  getByAssetId(id: string) {
+    return this.db.selectFrom('asset_ocr').selectAll('asset_ocr').where('asset_ocr.assetId', '=', id).execute();
+  }
+
+  deleteAll() {
+    return this.db.transaction().execute(async (trx: Kysely<DB>) => {
+      await sql`truncate ${sql.table('asset_ocr')}`.execute(trx);
+      await sql`truncate ${sql.table('ocr_search')}`.execute(trx);
+    });
+  }
+
+  @GenerateSql({
+    params: [
+      DummyValue.UUID,
+      [
+        {
+          assetId: DummyValue.UUID,
+          x1: DummyValue.NUMBER,
+          y1: DummyValue.NUMBER,
+          x2: DummyValue.NUMBER,
+          y2: DummyValue.NUMBER,
+          x3: DummyValue.NUMBER,
+          y3: DummyValue.NUMBER,
+          x4: DummyValue.NUMBER,
+          y4: DummyValue.NUMBER,
+          text: DummyValue.STRING,
+          boxScore: DummyValue.NUMBER,
+          textScore: DummyValue.NUMBER,
+        },
+      ],
+    ],
+  })
+  upsert(assetId: string, ocrDataList: Insertable<AssetOcrTable>[]) {
+    let query = this.db.with('deleted_ocr', (db) => db.deleteFrom('asset_ocr').where('assetId', '=', assetId));
+    if (ocrDataList.length > 0) {
+      const searchText = ocrDataList.map((item) => item.text.trim()).join(' ');
+      (query as any) = query
+        .with('inserted_ocr', (db) => db.insertInto('asset_ocr').values(ocrDataList))
+        .with('inserted_search', (db) =>
+          db
+            .insertInto('ocr_search')
+            .values({ assetId, text: searchText })
+            .onConflict((oc) => oc.column('assetId').doUpdateSet((eb) => ({ text: eb.ref('excluded.text') }))),
+        );
+    } else {
+      (query as any) = query.with('deleted_search', (db) => db.deleteFrom('ocr_search').where('assetId', '=', assetId));
+    }
+
+    return query.selectNoFrom(sql`1`.as('dummy')).execute();
+  }
+}

server/src/repositories/search.repository.ts

@@ -84,6 +84,10 @@ export interface SearchEmbeddingOptions {
   userIds: string[];
 }
 
+export interface SearchOcrOptions {
+  ocr?: string;
+}
+
 export interface SearchPeopleOptions {
   personIds?: string[];
 }
@@ -114,7 +118,8 @@ type BaseAssetSearchOptions = SearchDateOptions &
   SearchUserIdOptions &
   SearchPeopleOptions &
   SearchTagOptions &
-  SearchAlbumOptions;
+  SearchAlbumOptions &
+  SearchOcrOptions;
 
 export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions;
 
@@ -127,7 +132,10 @@ export type SmartSearchOptions = SearchDateOptions &
   SearchStatusOptions &
   SearchUserIdOptions &
   SearchPeopleOptions &
-  SearchTagOptions;
+  SearchTagOptions &
+  SearchOcrOptions;
+
+export type OcrSearchOptions = SearchDateOptions & SearchOcrOptions;
 
 export type LargeAssetSearchOptions = AssetSearchOptions & { minFileSize?: number };
 

server/src/schema/index.ts

@@ -35,6 +35,7 @@ import { AssetFileTable } from 'src/schema/tables/asset-file.table';
 import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table';
 import { AssetMetadataAuditTable } from 'src/schema/tables/asset-metadata-audit.table';
 import { AssetMetadataTable } from 'src/schema/tables/asset-metadata.table';
+import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
 import { AssetTable } from 'src/schema/tables/asset.table';
 import { AuditTable } from 'src/schema/tables/audit.table';
 import { FaceSearchTable } from 'src/schema/tables/face-search.table';
@@ -47,6 +48,7 @@ import { MemoryTable } from 'src/schema/tables/memory.table';
 import { MoveTable } from 'src/schema/tables/move.table';
 import { NaturalEarthCountriesTable } from 'src/schema/tables/natural-earth-countries.table';
 import { NotificationTable } from 'src/schema/tables/notification.table';
+import { OcrSearchTable } from 'src/schema/tables/ocr-search.table';
 import { PartnerAuditTable } from 'src/schema/tables/partner-audit.table';
 import { PartnerTable } from 'src/schema/tables/partner.table';
 import { PersonAuditTable } from 'src/schema/tables/person-audit.table';
@@ -87,6 +89,7 @@ export class ImmichDatabase {
     AssetMetadataTable,
     AssetMetadataAuditTable,
     AssetJobStatusTable,
+    AssetOcrTable,
     AssetTable,
     AssetFileTable,
     AuditTable,
@@ -101,6 +104,7 @@ export class ImmichDatabase {
     MoveTable,
     NaturalEarthCountriesTable,
     NotificationTable,
+    OcrSearchTable,
     PartnerAuditTable,
     PartnerTable,
     PersonTable,
@@ -174,6 +178,8 @@ export interface DB {
   asset_metadata: AssetMetadataTable;
   asset_metadata_audit: AssetMetadataAuditTable;
   asset_job_status: AssetJobStatusTable;
+  asset_ocr: AssetOcrTable;
+  ocr_search: OcrSearchTable;
 
   audit: AuditTable;
 

server/src/schema/migrations/1758705774125-CreateAssetOCRTable.ts

@@ -0,0 +1,16 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TABLE "asset_ocr" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "assetId" uuid NOT NULL, "x1" real NOT NULL, "y1" real NOT NULL, "x2" real NOT NULL, "y2" real NOT NULL, "x3" real NOT NULL, "y3" real NOT NULL, "x4" real NOT NULL, "y4" real NOT NULL, "boxScore" real NOT NULL, "textScore" real NOT NULL, "text" text NOT NULL);`.execute(
+    db,
+  );
+  await sql`ALTER TABLE "asset_ocr" ADD CONSTRAINT "asset_ocr_pkey" PRIMARY KEY ("id");`.execute(db);
+  await sql`ALTER TABLE "asset_ocr" ADD CONSTRAINT "asset_ocr_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(
+    db,
+  );
+  await sql`CREATE INDEX "asset_ocr_assetId_idx" ON "asset_ocr" ("assetId")`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TABLE "asset_ocr";`.execute(db);
+}

server/src/schema/migrations/1758705789125-CreateOCRSearchTable.ts

@@ -0,0 +1,20 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TABLE "ocr_search" ("assetId" uuid NOT NULL, "text" text NOT NULL);`.execute(db);
+  await sql`ALTER TABLE "ocr_search" ADD CONSTRAINT "ocr_search_pkey" PRIMARY KEY ("assetId");`.execute(db);
+  await sql`ALTER TABLE "ocr_search" ADD CONSTRAINT "ocr_search_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(
+    db,
+  );
+  await sql`CREATE INDEX "idx_ocr_search_text" ON "ocr_search" USING gin (f_unaccent("text") gin_trgm_ops);`.execute(
+    db,
+  );
+  await sql`INSERT INTO "migration_overrides" ("name", "value") VALUES ('index_idx_ocr_search_text', '{"type":"index","name":"idx_ocr_search_text","sql":"CREATE INDEX \\"idx_ocr_search_text\\" ON \\"ocr_search\\" USING gin (f_unaccent(\\"text\\") gin_trgm_ops);"}'::jsonb);`.execute(
+    db,
+  );
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TABLE "ocr_search";`.execute(db);
+  await sql`DELETE FROM "migration_overrides" WHERE "name" = 'index_idx_ocr_search_text';`.execute(db);
+}

server/src/schema/migrations/1758705804128-UpsertOcrAssetJobStatus.ts

@@ -0,0 +1,9 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset_job_status" ADD "ocrAt" timestamp with time zone;`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`ALTER TABLE "asset_job_status" DROP COLUMN "ocrAt";`.execute(db);
+}

server/src/schema/tables/asset-job-status.table.ts

@@ -20,4 +20,7 @@ export class AssetJobStatusTable {
 
   @Column({ type: 'timestamp with time zone', nullable: true })
   thumbnailAt!: Timestamp | null;
+
+  @Column({ type: 'timestamp with time zone', nullable: true })
+  ocrAt!: Timestamp | null;
 }

server/src/schema/tables/asset-ocr.table.ts

@@ -0,0 +1,45 @@
+import { AssetTable } from 'src/schema/tables/asset.table';
+import { Column, ForeignKeyColumn, Generated, PrimaryGeneratedColumn, Table } from 'src/sql-tools';
+
+@Table('asset_ocr')
+export class AssetOcrTable {
+  @PrimaryGeneratedColumn()
+  id!: Generated<string>;
+
+  @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
+  assetId!: string;
+
+  // box positions are normalized, with values between 0 and 1
+  @Column({ type: 'real' })
+  x1!: number;
+
+  @Column({ type: 'real' })
+  y1!: number;
+
+  @Column({ type: 'real' })
+  x2!: number;
+
+  @Column({ type: 'real' })
+  y2!: number;
+
+  @Column({ type: 'real' })
+  x3!: number;
+
+  @Column({ type: 'real' })
+  y3!: number;
+
+  @Column({ type: 'real' })
+  x4!: number;
+
+  @Column({ type: 'real' })
+  y4!: number;
+
+  @Column({ type: 'real' })
+  boxScore!: number;
+
+  @Column({ type: 'real' })
+  textScore!: number;
+
+  @Column({ type: 'text' })
+  text!: string;
+}

server/src/schema/tables/ocr-search.table.ts

@@ -0,0 +1,20 @@
+import { AssetTable } from 'src/schema/tables/asset.table';
+import { Column, ForeignKeyColumn, Index, Table } from 'src/sql-tools';
+
+@Table('ocr_search')
+@Index({
+  name: 'idx_ocr_search_text',
+  using: 'gin',
+  expression: 'f_unaccent("text") gin_trgm_ops',
+})
+export class OcrSearchTable {
+  @ForeignKeyColumn(() => AssetTable, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE',
+    primary: true,
+  })
+  assetId!: string;
+
+  @Column({ type: 'text' })
+  text!: string;
+}

server/src/services/base.service.ts

@@ -32,6 +32,7 @@ import { MetadataRepository } from 'src/repositories/metadata.repository';
 import { MoveRepository } from 'src/repositories/move.repository';
 import { NotificationRepository } from 'src/repositories/notification.repository';
 import { OAuthRepository } from 'src/repositories/oauth.repository';
+import { OcrRepository } from 'src/repositories/ocr.repository';
 import { PartnerRepository } from 'src/repositories/partner.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { ProcessRepository } from 'src/repositories/process.repository';
@@ -84,6 +85,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
   MoveRepository,
   NotificationRepository,
   OAuthRepository,
+  OcrRepository,
   PartnerRepository,
   PersonRepository,
   ProcessRepository,
@@ -137,6 +139,7 @@ export class BaseService {
     protected moveRepository: MoveRepository,
     protected notificationRepository: NotificationRepository,
     protected oauthRepository: OAuthRepository,
+    protected ocrRepository: OcrRepository,
     protected partnerRepository: PartnerRepository,
     protected personRepository: PersonRepository,
     protected processRepository: ProcessRepository,

server/src/services/index.ts

@@ -20,6 +20,7 @@ import { MemoryService } from 'src/services/memory.service';
 import { MetadataService } from 'src/services/metadata.service';
 import { NotificationAdminService } from 'src/services/notification-admin.service';
 import { NotificationService } from 'src/services/notification.service';
+import { OcrService } from 'src/services/ocr.service';
 import { PartnerService } from 'src/services/partner.service';
 import { PersonService } from 'src/services/person.service';
 import { SearchService } from 'src/services/search.service';
@@ -65,6 +66,7 @@ export const services = [
   MetadataService,
   NotificationService,
   NotificationAdminService,
+  OcrService,
   PartnerService,
   PersonService,
   SearchService,

server/src/services/job.service.spec.ts

@@ -24,7 +24,7 @@ describe(JobService.name, () => {
     it('should update concurrency', () => {
       sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
 
-      expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(15);
+      expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(16);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5);
@@ -98,6 +98,7 @@ describe(JobService.name, () => {
         [QueueName.Library]: expectedJobStatus,
         [QueueName.Notification]: expectedJobStatus,
         [QueueName.BackupDatabase]: expectedJobStatus,
+        [QueueName.Ocr]: expectedJobStatus,
       });
     });
   });
@@ -268,12 +269,12 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.AssetGenerateThumbnails, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces],
+        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.Ocr],
         stub: [assetStub.livePhotoStillAsset],
       },
       {
         item: { name: JobName.AssetGenerateThumbnails, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.AssetEncodeVideo],
+        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.Ocr, JobName.AssetEncodeVideo],
         stub: [assetStub.video],
       },
       {

server/src/services/job.service.ts

@@ -236,6 +236,10 @@ export class JobService extends BaseService {
         return this.jobRepository.queue({ name: JobName.DatabaseBackup, data: { force } });
       }
 
+      case QueueName.Ocr: {
+        return this.jobRepository.queue({ name: JobName.OcrQueueAll, data: { force } });
+      }
+
       default: {
         throw new BadRequestException(`Invalid job name: ${name}`);
       }
@@ -350,6 +354,7 @@ export class JobService extends BaseService {
         const jobs: JobItem[] = [
           { name: JobName.SmartSearch, data: item.data },
           { name: JobName.AssetDetectFaces, data: item.data },
+          { name: JobName.Ocr, data: item.data },
         ];
 
         if (asset.type === AssetType.Video) {

server/src/services/ocr.service.spec.ts

@@ -0,0 +1,177 @@
+import { AssetVisibility, ImmichWorker, JobName, JobStatus } from 'src/enum';
+import { OcrService } from 'src/services/ocr.service';
+import { assetStub } from 'test/fixtures/asset.stub';
+import { systemConfigStub } from 'test/fixtures/system-config.stub';
+import { makeStream, newTestService, ServiceMocks } from 'test/utils';
+
+describe(OcrService.name, () => {
+  let sut: OcrService;
+  let mocks: ServiceMocks;
+
+  beforeEach(() => {
+    ({ sut, mocks } = newTestService(OcrService));
+
+    mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);
+  });
+
+  it('should work', () => {
+    expect(sut).toBeDefined();
+  });
+
+  describe('handleQueueOcr', () => {
+    it('should do nothing if machine learning is disabled', async () => {
+      mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
+
+      await sut.handleQueueOcr({ force: false });
+
+      expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
+    });
+
+    it('should queue the assets without ocr', async () => {
+      mocks.assetJob.streamForOcrJob.mockReturnValue(makeStream([assetStub.image]));
+
+      await sut.handleQueueOcr({ force: false });
+
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.Ocr, data: { id: assetStub.image.id } }]);
+      expect(mocks.assetJob.streamForOcrJob).toHaveBeenCalledWith(false);
+    });
+
+    it('should queue all the assets', async () => {
+      mocks.assetJob.streamForOcrJob.mockReturnValue(makeStream([assetStub.image]));
+
+      await sut.handleQueueOcr({ force: true });
+
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.Ocr, data: { id: assetStub.image.id } }]);
+      expect(mocks.assetJob.streamForOcrJob).toHaveBeenCalledWith(true);
+    });
+  });
+
+  describe('handleOcr', () => {
+    it('should do nothing if machine learning is disabled', async () => {
+      mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
+
+      expect(await sut.handleOcr({ id: '123' })).toEqual(JobStatus.Skipped);
+
+      expect(mocks.asset.getByIds).not.toHaveBeenCalled();
+      expect(mocks.machineLearning.encodeImage).not.toHaveBeenCalled();
+    });
+
+    it('should skip assets without a resize path', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue({ visibility: AssetVisibility.Timeline, previewFile: null });
+
+      expect(await sut.handleOcr({ id: assetStub.noResizePath.id })).toEqual(JobStatus.Failed);
+
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+    });
+
+    it('should save the returned objects', async () => {
+      mocks.machineLearning.ocr.mockResolvedValue({
+        box: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160],
+        boxScore: [0.9, 0.8],
+        text: ['One Two Three', 'Four Five'],
+        textScore: [0.95, 0.85],
+      });
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Timeline,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Success);
+
+      expect(mocks.machineLearning.ocr).toHaveBeenCalledWith(
+        '/uploads/user-id/thumbs/path.jpg',
+        expect.objectContaining({
+          modelName: 'PP-OCRv5_mobile',
+          minDetectionScore: 0.5,
+          minRecognitionScore: 0.8,
+          maxResolution: 736,
+        }),
+      );
+      expect(mocks.ocr.upsert).toHaveBeenCalledWith(assetStub.image.id, [
+        {
+          assetId: assetStub.image.id,
+          boxScore: 0.9,
+          text: 'One Two Three',
+          textScore: 0.95,
+          x1: 10,
+          y1: 20,
+          x2: 30,
+          y2: 40,
+          x3: 50,
+          y3: 60,
+          x4: 70,
+          y4: 80,
+        },
+        {
+          assetId: assetStub.image.id,
+          boxScore: 0.8,
+          text: 'Four Five',
+          textScore: 0.85,
+          x1: 90,
+          y1: 100,
+          x2: 110,
+          y2: 120,
+          x3: 130,
+          y3: 140,
+          x4: 150,
+          y4: 160,
+        },
+      ]);
+    });
+
+    it('should apply config settings', async () => {
+      mocks.systemMetadata.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          ocr: {
+            modelName: 'PP-OCRv5_server',
+            enabled: true,
+            minDetectionScore: 0.8,
+            minRecognitionScore: 0.9,
+            maxResolution: 1500,
+          },
+        },
+      });
+      mocks.machineLearning.ocr.mockResolvedValue({ box: [], boxScore: [], text: [], textScore: [] });
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Timeline,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Success);
+
+      expect(mocks.machineLearning.ocr).toHaveBeenCalledWith(
+        '/uploads/user-id/thumbs/path.jpg',
+        expect.objectContaining({
+          modelName: 'PP-OCRv5_server',
+          minDetectionScore: 0.8,
+          minRecognitionScore: 0.9,
+          maxResolution: 1500,
+        }),
+      );
+      expect(mocks.ocr.upsert).toHaveBeenCalledWith(assetStub.image.id, []);
+    });
+
+    it('should skip invisible assets', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Hidden,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.Skipped);
+
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+    });
+
+    it('should fail if asset could not be found', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue(void 0);
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Failed);
+
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+    });
+  });
+});

server/src/services/ocr.service.ts

@@ -0,0 +1,86 @@
+import { Injectable } from '@nestjs/common';
+import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
+import { OnJob } from 'src/decorators';
+import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
+import { OCR } from 'src/repositories/machine-learning.repository';
+import { BaseService } from 'src/services/base.service';
+import { JobItem, JobOf } from 'src/types';
+import { isOcrEnabled } from 'src/utils/misc';
+
+@Injectable()
+export class OcrService extends BaseService {
+  @OnJob({ name: JobName.OcrQueueAll, queue: QueueName.Ocr })
+  async handleQueueOcr({ force }: JobOf<JobName.OcrQueueAll>): Promise<JobStatus> {
+    const { machineLearning } = await this.getConfig({ withCache: false });
+    if (!isOcrEnabled(machineLearning)) {
+      return JobStatus.Skipped;
+    }
+
+    if (force) {
+      await this.ocrRepository.deleteAll();
+    }
+
+    let jobs: JobItem[] = [];
+    const assets = this.assetJobRepository.streamForOcrJob(force);
+
+    for await (const asset of assets) {
+      jobs.push({ name: JobName.Ocr, data: { id: asset.id } });
+
+      if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
+        await this.jobRepository.queueAll(jobs);
+        jobs = [];
+      }
+    }
+
+    await this.jobRepository.queueAll(jobs);
+    return JobStatus.Success;
+  }
+
+  @OnJob({ name: JobName.Ocr, queue: QueueName.Ocr })
+  async handleOcr({ id }: JobOf<JobName.Ocr>): Promise<JobStatus> {
+    const { machineLearning } = await this.getConfig({ withCache: true });
+    if (!isOcrEnabled(machineLearning)) {
+      return JobStatus.Skipped;
+    }
+
+    const asset = await this.assetJobRepository.getForOcr(id);
+    if (!asset || !asset.previewFile) {
+      return JobStatus.Failed;
+    }
+
+    if (asset.visibility === AssetVisibility.Hidden) {
+      return JobStatus.Skipped;
+    }
+
+    const ocrResults = await this.machineLearningRepository.ocr(asset.previewFile, machineLearning.ocr);
+
+    await this.ocrRepository.upsert(id, this.parseOcrResults(id, ocrResults));
+
+    await this.assetRepository.upsertJobStatus({ assetId: id, ocrAt: new Date() });
+
+    this.logger.debug(`Processed ${ocrResults.text.length} OCR result(s) for ${id}`);
+    return JobStatus.Success;
+  }
+
+  private parseOcrResults(id: string, { box, boxScore, text, textScore }: OCR) {
+    const ocrDataList = [];
+    for (let i = 0; i < text.length; i++) {
+      const boxOffset = i * 8;
+      ocrDataList.push({
+        assetId: id,
+        x1: box[boxOffset],
+        y1: box[boxOffset + 1],
+        x2: box[boxOffset + 2],
+        y2: box[boxOffset + 3],
+        x3: box[boxOffset + 4],
+        y3: box[boxOffset + 5],
+        x4: box[boxOffset + 6],
+        y4: box[boxOffset + 7],
+        boxScore: boxScore[i],
+        textScore: textScore[i],
+        text: text[i],
+      });
+    }
+    return ocrDataList;
+  }
+}

server/src/services/server.service.spec.ts

@@ -141,6 +141,7 @@ describe(ServerService.name, () => {
         reverseGeocoding: true,
         oauth: false,
         oauthAutoLaunch: false,
+        ocr: true,
         passwordLogin: true,
         search: true,
         sidecar: true,

server/src/services/server.service.ts

@@ -19,7 +19,12 @@ import { UserStatsQueryResponse } from 'src/repositories/user.repository';
 import { BaseService } from 'src/services/base.service';
 import { asHumanReadable } from 'src/utils/bytes';
 import { mimeTypes } from 'src/utils/mime-types';
-import { isDuplicateDetectionEnabled, isFacialRecognitionEnabled, isSmartSearchEnabled } from 'src/utils/misc';
+import {
+  isDuplicateDetectionEnabled,
+  isFacialRecognitionEnabled,
+  isOcrEnabled,
+  isSmartSearchEnabled,
+} from 'src/utils/misc';
 
 @Injectable()
 export class ServerService extends BaseService {
@@ -97,6 +102,7 @@ export class ServerService extends BaseService {
       trash: trash.enabled,
       oauth: oauth.enabled,
       oauthAutoLaunch: oauth.autoLaunch,
+      ocr: isOcrEnabled(machineLearning),
       passwordLogin: passwordLogin.enabled,
       configFile: !!configFile,
       email: notifications.smtp.enabled,

server/src/services/system-config.service.spec.ts

@@ -39,6 +39,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     [QueueName.ThumbnailGeneration]: { concurrency: 3 },
     [QueueName.VideoConversion]: { concurrency: 1 },
     [QueueName.Notification]: { concurrency: 5 },
+    [QueueName.Ocr]: { concurrency: 1 },
   },
   backup: {
     database: {
@@ -102,6 +103,13 @@ const updatedConfig = Object.freeze<SystemConfig>({
       maxDistance: 0.5,
       minFaces: 3,
     },
+    ocr: {
+      enabled: true,
+      modelName: 'PP-OCRv5_mobile',
+      minDetectionScore: 0.5,
+      minRecognitionScore: 0.8,
+      maxResolution: 736,
+    },
   },
   map: {
     enabled: true,

server/src/sql-tools/types.ts

@@ -322,7 +322,8 @@ export type ColumnType =
   | 'uuid'
   | 'vector'
   | 'enum'
-  | 'serial';
+  | 'serial'
+  | 'real';
 
 export type DatabaseSchema = {
   databaseName: string;

server/src/types.ts

@@ -370,7 +370,11 @@ export type JobItem =
   | { name: JobName.NotifyUserSignup; data: INotifySignupJob }
 
   // Version check
-  | { name: JobName.VersionCheck; data: IBaseJob };
+  | { name: JobName.VersionCheck; data: IBaseJob }
+
+  // OCR
+  | { name: JobName.OcrQueueAll; data: IBaseJob }
+  | { name: JobName.Ocr; data: IEntityJob };
 
 export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];
 

server/src/utils/database.ts

@@ -200,6 +200,14 @@ export function withFiles(eb: ExpressionBuilder<DB, 'asset'>, type?: AssetFileTy
   ).as('files');
 }
 
+export function withFilePath(eb: ExpressionBuilder<DB, 'asset'>, type: AssetFileType) {
+  return eb
+    .selectFrom('asset_file')
+    .select('asset_file.path')
+    .whereRef('asset_file.assetId', '=', 'asset.id')
+    .where('asset_file.type', '=', type);
+}
+
 export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
   return jsonArrayFrom(
     eb
@@ -380,6 +388,11 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
         .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
         .where(sql`f_unaccent(asset_exif.description)`, 'ilike', sql`'%' || f_unaccent(${options.description}) || '%'`),
     )
+    .$if(!!options.ocr, (qb) =>
+      qb
+        .innerJoin('ocr_search', 'asset.id', 'ocr_search.assetId')
+        .where(() => sql`f_unaccent(ocr_search.text) %>> f_unaccent(${options.ocr!})`),
+    )
     .$if(!!options.type, (qb) => qb.where('asset.type', '=', options.type!))
     .$if(options.isFavorite !== undefined, (qb) => qb.where('asset.isFavorite', '=', options.isFavorite!))
     .$if(options.isOffline !== undefined, (qb) => qb.where('asset.isOffline', '=', options.isOffline!))

server/src/utils/misc.ts

@@ -95,6 +95,8 @@ export const unsetDeep = (object: unknown, key: string) => {
 const isMachineLearningEnabled = (machineLearning: SystemConfig['machineLearning']) => machineLearning.enabled;
 export const isSmartSearchEnabled = (machineLearning: SystemConfig['machineLearning']) =>
   isMachineLearningEnabled(machineLearning) && machineLearning.clip.enabled;
+export const isOcrEnabled = (machineLearning: SystemConfig['machineLearning']) =>
+  isMachineLearningEnabled(machineLearning) && machineLearning.ocr.enabled;
 export const isFacialRecognitionEnabled = (machineLearning: SystemConfig['machineLearning']) =>
   isMachineLearningEnabled(machineLearning) && machineLearning.facialRecognition.enabled;
 export const isDuplicateDetectionEnabled = (machineLearning: SystemConfig['machineLearning']) =>