immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-02-05 00:30:57 +03:00
Code Issues 13 Packages Projects Releases 10 Wiki Activity

[BUG] Migration job throws error if file already moved #1412

New Issue
Closed
opened 2026-02-05 01:42:48 +03:00 by OVERLORD · 1 comment
OVERLORD commented 2026-02-05 01:42:48 +03:00
Owner
Copy Link

Originally created by @EnochPrime on GitHub (Oct 4, 2023).

The bug

Running the migration job a second time (was not completed the first time due to microservices OOM) reports many errors about the new thumbnail file path already existing. After the job completes there are still thumbnails in the folder not sorted into the new sub-folders.

immich_microservices.1.junnxzjzgf5p@McMint    | [Nest] 8  - 10/04/2023, 11:14:35 AM   ERROR [JobService] Unable to run job handler (migration/migrate-asset): Error: Destination file already exists: upload/thumbs/c75191a1-e351-418b-89a2-2cf5fd04633e/37/56/37562e4e-ecf5-453f-85ce-26333bd2a57c.jpeg
immich_microservices.1.junnxzjzgf5p@McMint    | [Nest] 8  - 10/04/2023, 11:14:35 AM   ERROR [JobService] Error: Destination file already exists: upload/thumbs/c75191a1-e351-418b-89a2-2cf5fd04633e/37/56/37562e4e-ecf5-453f-85ce-26333bd2a57c.jpeg
immich_microservices.1.junnxzjzgf5p@McMint    |     at FilesystemProvider.moveFile (/usr/src/app/dist/infra/repositories/filesystem.provider.js:74:19)
immich_microservices.1.junnxzjzgf5p@McMint    |     at async MediaService.handleAssetMigration (/usr/src/app/dist/domain/media/media.service.js:131:17)
immich_microservices.1.junnxzjzgf5p@McMint    |     at async /usr/src/app/dist/domain/job/job.service.js:109:37
immich_microservices.1.junnxzjzgf5p@McMint    |     at async Worker.processJob (/usr/src/app/node_modules/bullmq/dist/cjs/classes/worker.js:346:28)
immich_microservices.1.junnxzjzgf5p@McMint    |     at async Worker.retryIfFailed (/usr/src/app/node_modules/bullmq/dist/cjs/classes/worker.js:531:24)

The OS that Immich Server is running on

Linux Mint

Version of Immich Server

v1.81.0

Version of Immich Mobile App

v1.80.0

Platform with the issue

  • Server
  • Web
  • Mobile

Your docker-compose.yml content

version: "3.9"

# swarm deployment extension
x-deploy-swarm: &deploy_swarm
  mode: replicated
  replicas: 1
  update_config:
    parallelism: 1
    delay: 10s
    order: stop-first

# immich environment
x-immich-env: &immich_environment
  NODE_ENV: production
  DB_HOSTNAME: db
  DB_DATABASE_NAME: ${IMMICH_DB:-immich}
  DB_USERNAME: ${IMMICH_DB_USERNAME:-immich}
  DB_PASSWORD: ${IMMICH_DB_PASSWORD:-immich}
  TYPESENSE_HOST: typesense
  TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:?error}

services:
  app:
    image: ghcr.io/immich-app/immich-web:v1.80.0
    environment:
      IMMICH_SERVER_URL: http://server:3001
    networks:
      - frontend
    deploy:
      <<: *deploy_swarm
      resources:
        limits:
          cpus: ${IMMICH_APP_CPU_LIMIT}
          memory: ${IMMICH_APP_MEM_LIMIT}

  server:
    image: ghcr.io/immich-app/immich-server:v1.80.0
    command: start-server.sh
    environment:
      <<: *immich_environment
    volumes:
      - /media/pictures/immich:/usr/src/app/upload/library
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/upload:/usr/src/app/upload/upload
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/profile:/usr/src/app/upload/profile
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/thumbs:/usr/src/app/upload/thumbs
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/encoded-video:/usr/src/app/upload/encoded-video
      - /media/pictures:/media/pictures:ro
    networks:
      - frontend
      - database
      - machine-learning
      - redis
      - typesense
    deploy:
      <<: *deploy_swarm
      resources:
        limits:
          cpus: ${IMMICH_SERVER_CPU_LIMIT}
          memory: ${IMMICH_SERVER_MEM_LIMIT}
      restart_policy:
        delay: 120s
        max_attempts: 3
      placement:
        constraints:
          - node.hostname == McMint

  microservices:
    image: ghcr.io/immich-app/immich-server:v1.80.0
    command: start-microservices.sh
    environment:
      <<: *immich_environment
    volumes:
      - /media/pictures/immich:/usr/src/app/upload/library
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/upload:/usr/src/app/upload/upload
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/profile:/usr/src/app/upload/profile
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/thumbs:/usr/src/app/upload/thumbs
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/encoded-video:/usr/src/app/upload/encoded-video
      - /media/pictures:/media/pictures:ro
    networks:
      - database
      - machine-learning
      - redis
      - typesense
    deploy:
      <<: *deploy_swarm
      resources:
        limits:
          cpus: ${IMMICH_MICROSERVICES_CPU_LIMIT}
          memory: ${IMMICH_MICROSERVICES_MEM_LIMIT}
      restart_policy:
        delay: 120s
        max_attempts: 3
      placement:
        constraints:
          - node.hostname == McMint

  machine-learning:
    image: ghcr.io/immich-app/immich-machine-learning:v1.80.0
    volumes:
      - ${STORAGE_PATH}/immich/ml-cache:/cache
    networks:
      - machine-learning
    deploy:
      <<: *deploy_swarm
      placement:
        constraints:
          - node.platform.arch == x86_64
      resources:
        limits:
          cpus: ${IMMICH_ML_CPU_LIMIT}
          memory: ${IMMICH_ML_MEM_LIMIT}

  db:
    image: postgres:14.8
    environment:
      POSTGRES_DB: ${IMMICH_DB:-immich}
      POSTGRES_USER: ${IMMICH_DB_USERNAME:-immich}
      POSTGRES_PASSWORD: ${IMMICH_DB_PASSWORD:-immich}
    volumes:
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/db:/var/lib/postgresql/data
    networks:
      - database
    deploy:
      <<: *deploy_swarm
      resources:
        limits:
          cpus: ${IMMICH_DB_CPU_LIMIT}
          memory: ${IMMICH_DB_MEM_LIMIT}

  redis:
    image: redis:6.2.13
    networks:
      - redis
    deploy:
      <<: *deploy_swarm
      resources:
        limits:
          cpus: '0.1'
          memory: 32M

  typesense:
    image: typesense/typesense:0.24.1
    environment:
      TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:?error}
      TYPESENSE_DATA_DIR: /data
    volumes:
      - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/typesense:/data
    networks:
      - typesense
    deploy:
      <<: *deploy_swarm
      placement:
        constraints:
          - node.platform.arch == x86_64

networks:
  frontend:
    external: true
    name: core_frontend
  database:
    driver: overlay
  machine-learning:
    driver: overlay
  redis:
    driver: overlay
  typesense:
    driver: overlay

Your .env content

IMMICH_APP_CPU_LIMIT = '0.3'
IMMICH_APP_MEM_LIMIT = '128M'

IMMICH_DB_CPU_LIMIT = '0.3'
IMMICH_DB_MEM_LIMIT = '256M'

IMMICH_SERVER_CPU_LIMIT = '1.0'
IMMICH_SERVER_MEM_LIMIT = '256M'

IMMICH_MICROSERVICES_CPU_LIMIT = '4.0'
IMMICH_MICROSERVICES_MEM_LIMIT = '2.0G'

IMMICH_ML_CPU_LIMIT = '4.0'
IMMICH_ML_MEM_LIMIT = '3.0G'

Reproduction steps

1. Run migration job
2. Stop job in the middle by stopping services
3. Restart services
4. Job migration job again

Additional information

No response

Originally created by @EnochPrime on GitHub (Oct 4, 2023). ### The bug Running the migration job a second time (was not completed the first time due to microservices OOM) reports many errors about the new thumbnail file path already existing. After the job completes there are still thumbnails in the folder not sorted into the new sub-folders. ``` immich_microservices.1.junnxzjzgf5p@McMint | [Nest] 8 - 10/04/2023, 11:14:35 AM ERROR [JobService] Unable to run job handler (migration/migrate-asset): Error: Destination file already exists: upload/thumbs/c75191a1-e351-418b-89a2-2cf5fd04633e/37/56/37562e4e-ecf5-453f-85ce-26333bd2a57c.jpeg immich_microservices.1.junnxzjzgf5p@McMint | [Nest] 8 - 10/04/2023, 11:14:35 AM ERROR [JobService] Error: Destination file already exists: upload/thumbs/c75191a1-e351-418b-89a2-2cf5fd04633e/37/56/37562e4e-ecf5-453f-85ce-26333bd2a57c.jpeg immich_microservices.1.junnxzjzgf5p@McMint | at FilesystemProvider.moveFile (/usr/src/app/dist/infra/repositories/filesystem.provider.js:74:19) immich_microservices.1.junnxzjzgf5p@McMint | at async MediaService.handleAssetMigration (/usr/src/app/dist/domain/media/media.service.js:131:17) immich_microservices.1.junnxzjzgf5p@McMint | at async /usr/src/app/dist/domain/job/job.service.js:109:37 immich_microservices.1.junnxzjzgf5p@McMint | at async Worker.processJob (/usr/src/app/node_modules/bullmq/dist/cjs/classes/worker.js:346:28) immich_microservices.1.junnxzjzgf5p@McMint | at async Worker.retryIfFailed (/usr/src/app/node_modules/bullmq/dist/cjs/classes/worker.js:531:24) ``` ### The OS that Immich Server is running on Linux Mint ### Version of Immich Server v1.81.0 ### Version of Immich Mobile App v1.80.0 ### Platform with the issue - [X] Server - [ ] Web - [ ] Mobile ### Your docker-compose.yml content ```YAML version: "3.9" # swarm deployment extension x-deploy-swarm: &deploy_swarm mode: replicated replicas: 1 update_config: parallelism: 1 delay: 10s order: stop-first # immich environment x-immich-env: &immich_environment NODE_ENV: production DB_HOSTNAME: db DB_DATABASE_NAME: ${IMMICH_DB:-immich} DB_USERNAME: ${IMMICH_DB_USERNAME:-immich} DB_PASSWORD: ${IMMICH_DB_PASSWORD:-immich} TYPESENSE_HOST: typesense TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:?error} services: app: image: ghcr.io/immich-app/immich-web:v1.80.0 environment: IMMICH_SERVER_URL: http://server:3001 networks: - frontend deploy: <<: *deploy_swarm resources: limits: cpus: ${IMMICH_APP_CPU_LIMIT} memory: ${IMMICH_APP_MEM_LIMIT} server: image: ghcr.io/immich-app/immich-server:v1.80.0 command: start-server.sh environment: <<: *immich_environment volumes: - /media/pictures/immich:/usr/src/app/upload/library - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/upload:/usr/src/app/upload/upload - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/profile:/usr/src/app/upload/profile - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/thumbs:/usr/src/app/upload/thumbs - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/encoded-video:/usr/src/app/upload/encoded-video - /media/pictures:/media/pictures:ro networks: - frontend - database - machine-learning - redis - typesense deploy: <<: *deploy_swarm resources: limits: cpus: ${IMMICH_SERVER_CPU_LIMIT} memory: ${IMMICH_SERVER_MEM_LIMIT} restart_policy: delay: 120s max_attempts: 3 placement: constraints: - node.hostname == McMint microservices: image: ghcr.io/immich-app/immich-server:v1.80.0 command: start-microservices.sh environment: <<: *immich_environment volumes: - /media/pictures/immich:/usr/src/app/upload/library - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/upload:/usr/src/app/upload/upload - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/profile:/usr/src/app/upload/profile - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/thumbs:/usr/src/app/upload/thumbs - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/encoded-video:/usr/src/app/upload/encoded-video - /media/pictures:/media/pictures:ro networks: - database - machine-learning - redis - typesense deploy: <<: *deploy_swarm resources: limits: cpus: ${IMMICH_MICROSERVICES_CPU_LIMIT} memory: ${IMMICH_MICROSERVICES_MEM_LIMIT} restart_policy: delay: 120s max_attempts: 3 placement: constraints: - node.hostname == McMint machine-learning: image: ghcr.io/immich-app/immich-machine-learning:v1.80.0 volumes: - ${STORAGE_PATH}/immich/ml-cache:/cache networks: - machine-learning deploy: <<: *deploy_swarm placement: constraints: - node.platform.arch == x86_64 resources: limits: cpus: ${IMMICH_ML_CPU_LIMIT} memory: ${IMMICH_ML_MEM_LIMIT} db: image: postgres:14.8 environment: POSTGRES_DB: ${IMMICH_DB:-immich} POSTGRES_USER: ${IMMICH_DB_USERNAME:-immich} POSTGRES_PASSWORD: ${IMMICH_DB_PASSWORD:-immich} volumes: - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/db:/var/lib/postgresql/data networks: - database deploy: <<: *deploy_swarm resources: limits: cpus: ${IMMICH_DB_CPU_LIMIT} memory: ${IMMICH_DB_MEM_LIMIT} redis: image: redis:6.2.13 networks: - redis deploy: <<: *deploy_swarm resources: limits: cpus: '0.1' memory: 32M typesense: image: typesense/typesense:0.24.1 environment: TYPESENSE_API_KEY: ${TYPESENSE_API_KEY:?error} TYPESENSE_DATA_DIR: /data volumes: - ${STORAGE_PATH}/{{index .Service.Labels "com.docker.stack.namespace"}}/typesense:/data networks: - typesense deploy: <<: *deploy_swarm placement: constraints: - node.platform.arch == x86_64 networks: frontend: external: true name: core_frontend database: driver: overlay machine-learning: driver: overlay redis: driver: overlay typesense: driver: overlay ``` ### Your .env content ```Shell IMMICH_APP_CPU_LIMIT = '0.3' IMMICH_APP_MEM_LIMIT = '128M' IMMICH_DB_CPU_LIMIT = '0.3' IMMICH_DB_MEM_LIMIT = '256M' IMMICH_SERVER_CPU_LIMIT = '1.0' IMMICH_SERVER_MEM_LIMIT = '256M' IMMICH_MICROSERVICES_CPU_LIMIT = '4.0' IMMICH_MICROSERVICES_MEM_LIMIT = '2.0G' IMMICH_ML_CPU_LIMIT = '4.0' IMMICH_ML_MEM_LIMIT = '3.0G' ``` ### Reproduction steps ```bash 1. Run migration job 2. Stop job in the middle by stopping services 3. Restart services 4. Job migration job again ``` ### Additional information _No response_
OVERLORD commented 2026-02-05 01:42:53 +03:00
Author
Owner
Copy Link

@jrasm91 commented on GitHub (Nov 23, 2023):

I believe these issues have been fixed

@jrasm91 commented on GitHub (Nov 23, 2023): I believe these issues have been fixed
Sign in to join this conversation.
No Branch/Tag Specified
Branches Tags
Labels
Clear labels
accessibility
changelog:enhancement
changelog:security
changelog:translation
cli
date-time
documentation
external-library
format
good first issue
needs-answer
nice to have
sharing
tech-debt
📱mobile
🖥️web
🗄️server
🧠machine-learning
No Label
Milestone
No items
No Milestone
Projects
Clear projects
No project
Assignees
Clear assignees
OVERLORD
No Assignees
1 Participants
Notifications
Due Date
No due date set.
Dependencies

No dependencies set.

Reference: immich-app/immich#1412
Delete Branch "%!s()"

Deleting a branch is permanent. Although the deleted branch may continue to exist for a short time before it actually gets removed, it CANNOT be undone in most cases. Continue?