feat: show OCR bounding box (#23717)

* feat: ocr bounding box

* bounding boxes

* pr feedback

* pr feedback

* allow copy across text boxes

* pr feedback

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -13,6 +13,7 @@
   import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
   import { closeEditorCofirm } from '$lib/stores/asset-editor.store';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
+  import { ocrManager } from '$lib/stores/ocr.svelte';
   import { alwaysLoadOriginalVideo, isShowDetail } from '$lib/stores/preferences.store';
   import { SlideshowNavigation, SlideshowState, slideshowStore } from '$lib/stores/slideshow.store';
   import { user } from '$lib/stores/user.store';
@@ -44,6 +45,7 @@
   import CropArea from './editor/crop-tool/crop-area.svelte';
   import EditorPanel from './editor/editor-panel.svelte';
   import ImagePanoramaViewer from './image-panorama-viewer.svelte';
+  import OcrButton from './ocr-button.svelte';
   import PhotoViewer from './photo-viewer.svelte';
   import SlideshowBar from './slideshow-bar.svelte';
   import VideoViewer from './video-wrapper-viewer.svelte';
@@ -392,9 +394,13 @@
       handlePromiseError(activityManager.init(album.id, asset.id));
     }
   });
+
+  let currentAssetId = $derived(asset.id);
   $effect(() => {
-    if (asset.id) {
-      handlePromiseError(handleGetAllAlbums());
+    if (currentAssetId) {
+      untrack(() => handlePromiseError(handleGetAllAlbums()));
+      ocrManager.clear();
+      handlePromiseError(ocrManager.getAssetOcr(currentAssetId));
     }
   });
 </script>
@@ -535,6 +541,7 @@
             {playOriginalVideo}
           />
         {/if}
+
         {#if $slideshowState === SlideshowState.None && isShared && ((album && album.isActivityEnabled) || activityManager.commentCount > 0) && !activityManager.isLoading}
           <div class="absolute bottom-0 end-0 mb-20 me-8">
             <ActivityStatus
@@ -547,6 +554,12 @@
             />
           </div>
         {/if}
+
+        {#if $slideshowState === SlideshowState.None && asset.type === AssetTypeEnum.Image && !isShowEditor && ocrManager.hasOcrData}
+          <div class="absolute bottom-0 end-0 mb-6 me-6">
+            <OcrButton />
+          </div>
+        {/if}
       {/key}
     {/if}
   </div>

web/src/lib/components/asset-viewer/detail-panel.svelte

@@ -503,7 +503,7 @@
 {/if}
 
 {#if albums.length > 0}
-  <section class="px-6 pt-6 dark:text-immich-dark-fg">
+  <section class="px-6 py-6 dark:text-immich-dark-fg">
     <p class="uppercase pb-4 text-sm">{$t('appears_in')}</p>
     {#each albums as album (album.id)}
       <a href={resolve(`${AppRoute.ALBUMS}/${album.id}`)}>

web/src/lib/components/asset-viewer/ocr-bounding-box.svelte

@@ -0,0 +1,36 @@
+<script lang="ts">
+  import type { OcrBox } from '$lib/utils/ocr-utils';
+  import { calculateBoundingBoxDimensions } from '$lib/utils/ocr-utils';
+
+  type Props = {
+    ocrBox: OcrBox;
+  };
+
+  let { ocrBox }: Props = $props();
+
+  const dimensions = $derived(calculateBoundingBoxDimensions(ocrBox.points));
+
+  const transform = $derived(
+    `translate(${dimensions.minX}px, ${dimensions.minY}px) rotate(${dimensions.rotation}deg) skew(${dimensions.skewX}deg, ${dimensions.skewY}deg)`,
+  );
+
+  const transformOrigin = $derived(
+    `${dimensions.centerX - dimensions.minX}px ${dimensions.centerY - dimensions.minY}px`,
+  );
+</script>
+
+<div class="absolute group left-0 top-0 pointer-events-none">
+  <!-- Bounding box with CSS transforms -->
+  <div
+    class="absolute border-2 border-blue-500 bg-blue-500/10 cursor-pointer pointer-events-auto transition-all group-hover:bg-blue-500/30 group-hover:border-blue-600 group-hover:border-[3px]"
+    style="width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: {transformOrigin};"
+  ></div>
+
+  <!-- Text overlay - always rendered but invisible, allows text selection and copy -->
+  <div
+    class="absolute flex items-center justify-center text-transparent text-sm px-2 py-1 pointer-events-auto cursor-text whitespace-pre-wrap wrap-break-word select-text group-hover:text-white group-hover:bg-black/75 group-hover:z-10"
+    style="width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: {transformOrigin};"
+  >
+    {ocrBox.text}
+  </div>
+</div>

web/src/lib/components/asset-viewer/ocr-button.svelte

@@ -0,0 +1,17 @@
+<script lang="ts">
+  import { ocrManager } from '$lib/stores/ocr.svelte';
+  import { IconButton } from '@immich/ui';
+  import { mdiTextRecognition } from '@mdi/js';
+  import { t } from 'svelte-i18n';
+</script>
+
+<IconButton
+  title={ocrManager.showOverlay ? $t('hide_text_recognition') : $t('show_text_recognition')}
+  icon={mdiTextRecognition}
+  class={"dark {ocrStore.showOverlay ? 'bg-immich-primary text-white dark' : 'dark'}"}
+  color="secondary"
+  variant="ghost"
+  shape="round"
+  aria-label={$t('text_recognition')}
+  onclick={() => ocrManager.toggleOcrBoundingBox()}
+/>

web/src/lib/components/asset-viewer/photo-viewer.svelte

@@ -2,12 +2,14 @@
   import { shortcuts } from '$lib/actions/shortcut';
   import { zoomImageAction } from '$lib/actions/zoom-image';
   import FaceEditor from '$lib/components/asset-viewer/face-editor/face-editor.svelte';
+  import OcrBoundingBox from '$lib/components/asset-viewer/ocr-bounding-box.svelte';
   import BrokenAsset from '$lib/components/assets/broken-asset.svelte';
   import { assetViewerFadeDuration } from '$lib/constants';
   import { castManager } from '$lib/managers/cast-manager.svelte';
   import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
   import { photoViewerImgElement } from '$lib/stores/assets-store.svelte';
   import { isFaceEditMode } from '$lib/stores/face-edit.svelte';
+  import { ocrManager } from '$lib/stores/ocr.svelte';
   import { boundingBoxesArray } from '$lib/stores/people.store';
   import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
   import { SlideshowLook, SlideshowState, slideshowLookCssMapping, slideshowStore } from '$lib/stores/slideshow.store';
@@ -15,6 +17,7 @@
   import { getAssetOriginalUrl, getAssetThumbnailUrl, handlePromiseError } from '$lib/utils';
   import { canCopyImageToClipboard, copyImageToClipboard, isWebCompatibleImage } from '$lib/utils/asset-utils';
   import { handleError } from '$lib/utils/handle-error';
+  import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';
   import { getBoundingBox } from '$lib/utils/people-utils';
   import { cancelImageUrl } from '$lib/utils/sw-messaging';
   import { getAltText } from '$lib/utils/thumbnail-util';
@@ -71,6 +74,14 @@
     $boundingBoxesArray = [];
   });
 
+  let ocrBoxes = $derived(
+    ocrManager.showOverlay && $photoViewerImgElement
+      ? getOcrBoundingBoxes(ocrManager.data, $photoZoomState, $photoViewerImgElement)
+      : [],
+  );
+
+  let isOcrActive = $derived(ocrManager.showOverlay);
+
   const preload = (targetSize: AssetMediaSize | 'original', preloadAssets?: TimelineAsset[]) => {
     for (const preloadAsset of preloadAssets || []) {
       if (preloadAsset.isImage) {
@@ -130,9 +141,15 @@
     if ($photoZoomState.currentZoom > 1) {
       return;
     }
+
+    if (ocrManager.showOverlay) {
+      return;
+    }
+
     if (onNextAsset && event.detail.direction === 'left') {
       onNextAsset();
     }
+
     if (onPreviousAsset && event.detail.direction === 'right') {
       onPreviousAsset();
     }
@@ -235,7 +252,7 @@
     </div>
   {:else if !imageError}
     <div
-      use:zoomImageAction
+      use:zoomImageAction={{ disabled: isOcrActive }}
       {...useSwipe(onSwipe)}
       class="h-full w-full"
       transition:fade={{ duration: haveFadeTransition ? assetViewerFadeDuration : 0 }}
@@ -264,6 +281,10 @@
           style="top: {boundingbox.top}px; left: {boundingbox.left}px; height: {boundingbox.height}px; width: {boundingbox.width}px;"
         ></div>
       {/each}
+
+      {#each ocrBoxes as ocrBox (ocrBox.id)}
+        <OcrBoundingBox {ocrBox} />
+      {/each}
     </div>
 
     {#if isFaceEditMode.value}