Files
immich/server/src/immich/api-v1/asset/asset.service.ts

705 lines
24 KiB
TypeScript
Raw Normal View History

import {
AccessCore,
AssetResponseDto,
ASSET_MIME_TYPES,
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
AuthUserDto,
getLivePhotoMotionFilename,
IAccessRepository,
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
ICryptoRepository,
IJobRepository,
IStorageRepository,
JobName,
LIVE_PHOTO_MIME_TYPES,
mapAsset,
mapAssetWithoutExif,
Permission,
PROFILE_MIME_TYPES,
SIDECAR_MIME_TYPES,
StorageCore,
StorageFolder,
UploadFieldName,
UploadFile,
} from '@app/domain';
import { AssetEntity, AssetType } from '@app/infra/entities';
import {
BadRequestException,
Inject,
Injectable,
InternalServerErrorException,
Logger,
NotFoundException,
} from '@nestjs/common';
2022-02-03 10:06:44 -06:00
import { InjectRepository } from '@nestjs/typeorm';
import { Response as Res } from 'express';
import { constants, createReadStream } from 'fs';
import fs from 'fs/promises';
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
import mime from 'mime-types';
import path, { extname } from 'path';
import sanitize from 'sanitize-filename';
import { pipeline } from 'stream/promises';
import { QueryFailedError, Repository } from 'typeorm';
import { UploadRequest } from '../../app.interceptor';
import { IAssetRepository } from './asset-repository';
import { AssetCore } from './asset.core';
import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
import { AssetSearchDto } from './dto/asset-search.dto';
import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CreateAssetDto, ImportAssetDto } from './dto/create-asset.dto';
import { DeleteAssetDto } from './dto/delete-asset.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
import { GetAssetThumbnailDto, GetAssetThumbnailFormatEnum } from './dto/get-asset-thumbnail.dto';
import { SearchAssetDto } from './dto/search-asset.dto';
import { SearchPropertiesDto } from './dto/search-properties.dto';
import { ServeFileDto } from './dto/serve-file.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import {
AssetBulkUploadCheckResponseDto,
AssetRejectReason,
AssetUploadAction,
} from './response-dto/asset-check-response.dto';
import {
AssetCountByTimeBucketResponseDto,
mapAssetCountByTimeBucket,
} from './response-dto/asset-count-by-time-group-response.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
import { CheckDuplicateAssetResponseDto } from './response-dto/check-duplicate-asset-response.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
import { DeleteAssetResponseDto, DeleteAssetStatusEnum } from './response-dto/delete-asset-response.dto';
interface ServableFile {
filepath: string;
contentType: string;
}
2022-02-03 10:06:44 -06:00
@Injectable()
export class AssetService {
readonly logger = new Logger(AssetService.name);
private assetCore: AssetCore;
private access: AccessCore;
private storageCore = new StorageCore();
2022-02-03 10:06:44 -06:00
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.assetCore = new AssetCore(_assetRepository, jobRepository);
this.access = new AccessCore(accessRepository);
}
WIP refactor container and queuing system (#206) * refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
2022-06-11 16:12:06 -05:00
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {
this.access.requireUploadAccess(authUser);
switch (fieldName) {
case UploadFieldName.ASSET_DATA:
if (ASSET_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.LIVE_PHOTO_DATA:
if (LIVE_PHOTO_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.SIDECAR_DATA:
if (SIDECAR_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.PROFILE_DATA:
if (PROFILE_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
}
const ext = extname(file.originalName);
this.logger.error(`Unsupported file type ${ext} file MIME type ${file.mimeType}`);
throw new BadRequestException(`Unsupported file type ${ext}`);
}
getUploadFilename({ authUser, fieldName, file }: UploadRequest): string {
this.access.requireUploadAccess(authUser);
const originalExt = extname(file.originalName);
const lookup = {
[UploadFieldName.ASSET_DATA]: originalExt,
[UploadFieldName.LIVE_PHOTO_DATA]: '.mov',
[UploadFieldName.SIDECAR_DATA]: '.xmp',
[UploadFieldName.PROFILE_DATA]: originalExt,
};
return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`);
}
getUploadFolder({ authUser, fieldName }: UploadRequest): string {
authUser = this.access.requireUploadAccess(authUser);
let folder = this.storageCore.getFolderLocation(StorageFolder.UPLOAD, authUser.id);
if (fieldName === UploadFieldName.PROFILE_DATA) {
folder = this.storageCore.getFolderLocation(StorageFolder.PROFILE, authUser.id);
}
this.storageRepository.mkdirSync(folder);
return folder;
}
public async uploadFile(
authUser: AuthUserDto,
dto: CreateAssetDto,
file: UploadFile,
livePhotoFile?: UploadFile,
feat(server): xmp sidecar metadata (#2466) * initial commit for XMP sidecar support * Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards * didn't mean to commit default log level during testing * new sidecar logic for video metadata as well * Added xml mimetype for sidecars only * don't need capture group for this regex * wrong default value reverted * simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway * simplified setter logic Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> * simplified logic per suggestions * sidecar is now its own queue with a discover and sync, updated UI for the new job queueing * queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar * now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync * simplified logic of filename extraction and asset instantiation * not sure how that got deleted.. * updated code per suggestions and comments in the PR * stat was not being used, removed the variable set * better type checking, using in-scope variables for exif getter instead of passing in every time * removed commented out test * ran and resolved all lints, formats, checks, and tests * resolved suggested change in PR * made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking * better error handling and moving files back to positions on move or save failure * regenerated api * format fixes * Added XMP documentation * documentation typo * Merged in main * missed merge conflict * more changes due to a merge * Resolving conflicts * added icon for sidecar jobs --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-24 21:59:30 -04:00
sidecarFile?: UploadFile,
): Promise<AssetFileUploadResponseDto> {
if (livePhotoFile) {
livePhotoFile = {
...livePhotoFile,
originalName: getLivePhotoMotionFilename(file.originalName, livePhotoFile.originalName),
};
}
let livePhotoAsset: AssetEntity | null = null;
try {
if (livePhotoFile) {
const livePhotoDto = { ...dto, assetType: AssetType.VIDEO, isVisible: false };
livePhotoAsset = await this.assetCore.create(authUser, livePhotoDto, livePhotoFile);
}
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
const asset = await this.assetCore.create(authUser, dto, file, livePhotoAsset?.id, sidecarFile?.originalPath);
return { id: asset.id, duplicate: false };
} catch (error: any) {
// clean up files
await this.jobRepository.queue({
name: JobName.DELETE_FILES,
feat(server): xmp sidecar metadata (#2466) * initial commit for XMP sidecar support * Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards * didn't mean to commit default log level during testing * new sidecar logic for video metadata as well * Added xml mimetype for sidecars only * don't need capture group for this regex * wrong default value reverted * simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway * simplified setter logic Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> * simplified logic per suggestions * sidecar is now its own queue with a discover and sync, updated UI for the new job queueing * queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar * now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync * simplified logic of filename extraction and asset instantiation * not sure how that got deleted.. * updated code per suggestions and comments in the PR * stat was not being used, removed the variable set * better type checking, using in-scope variables for exif getter instead of passing in every time * removed commented out test * ran and resolved all lints, formats, checks, and tests * resolved suggested change in PR * made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking * better error handling and moving files back to positions on move or save failure * regenerated api * format fixes * Added XMP documentation * documentation typo * Merged in main * missed merge conflict * more changes due to a merge * Resolving conflicts * added icon for sidecar jobs --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-24 21:59:30 -04:00
data: { files: [file.originalPath, livePhotoFile?.originalPath, sidecarFile?.originalPath] },
});
// handle duplicates with a success response
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum);
const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
return { id: duplicate.id, duplicate: true };
}
this.logger.error(`Error uploading file ${error}`, error?.stack);
throw new BadRequestException(`Error uploading file`, `${error}`);
}
}
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
public async importFile(authUser: AuthUserDto, dto: ImportAssetDto): Promise<AssetFileUploadResponseDto> {
dto = {
...dto,
assetPath: path.resolve(dto.assetPath),
sidecarPath: dto.sidecarPath ? path.resolve(dto.sidecarPath) : undefined,
};
const mimeType = mime.lookup(dto.assetPath) as string;
if (!ASSET_MIME_TYPES.includes(mimeType)) {
throw new BadRequestException(`Unsupported file type ${mimeType}`);
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
}
if (dto.sidecarPath) {
if (path.extname(dto.sidecarPath).toLowerCase() !== '.xmp') {
throw new BadRequestException(`Unsupported sidecar file type`);
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
}
}
for (const filepath of [dto.assetPath, dto.sidecarPath]) {
if (!filepath) {
continue;
}
const exists = await this.storageRepository.checkFileExists(filepath, constants.R_OK);
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
if (!exists) {
throw new BadRequestException('File does not exist');
}
}
if (!authUser.externalPath || !dto.assetPath.match(new RegExp(`^${authUser.externalPath}`))) {
throw new BadRequestException("File does not exist within user's external path");
}
const assetFile: UploadFile = {
checksum: await this.cryptoRepository.hashFile(dto.assetPath),
mimeType,
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
originalPath: dto.assetPath,
originalName: path.parse(dto.assetPath).name,
};
try {
const asset = await this.assetCore.create(authUser, dto, assetFile, undefined, dto.sidecarPath);
return { id: asset.id, duplicate: false };
} catch (error: QueryFailedError | Error | any) {
// handle duplicates with a success response
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, [assetFile.checksum]);
return { id: duplicate.id, duplicate: true };
}
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_4ed4f8052685ff5b1e7ca1058ba') {
const duplicate = await this._assetRepository.getByOriginalPath(dto.assetPath);
if (duplicate) {
if (duplicate.ownerId === authUser.id) {
return { id: duplicate.id, duplicate: true };
}
throw new BadRequestException('Path in use by another user');
}
}
this.logger.error(`Error importing file ${error}`, error?.stack);
throw new BadRequestException(`Error importing file`, `${error}`);
}
}
2022-02-03 10:06:44 -06:00
public async getUserAssetsByDeviceId(authUser: AuthUserDto, deviceId: string) {
return this._assetRepository.getAllByDeviceId(authUser.id, deviceId);
2022-02-03 10:06:44 -06:00
}
public async getAllAssets(authUser: AuthUserDto, dto: AssetSearchDto): Promise<AssetResponseDto[]> {
const userId = dto.userId || authUser.id;
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
const assets = await this._assetRepository.getAllByUserId(userId, dto);
return assets.map((asset) => mapAsset(asset));
}
public async getAssetByTimeBucket(authUser: AuthUserDto, dto: GetAssetByTimeBucketDto): Promise<AssetResponseDto[]> {
const userId = dto.userId || authUser.id;
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
const assets = await this._assetRepository.getAssetByTimeBucket(userId, dto);
return assets.map((asset) => mapAsset(asset));
}
public async getAssetById(authUser: AuthUserDto, assetId: string): Promise<AssetResponseDto> {
await this.access.requirePermission(authUser, Permission.ASSET_READ, assetId);
const allowExif = this.getExifPermission(authUser);
const asset = await this._assetRepository.getById(assetId);
if (allowExif) {
return mapAsset(asset);
} else {
return mapAssetWithoutExif(asset);
}
}
2022-12-05 11:56:44 -06:00
public async updateAsset(authUser: AuthUserDto, assetId: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
await this.access.requirePermission(authUser, Permission.ASSET_UPDATE, assetId);
const asset = await this._assetRepository.getById(assetId);
if (!asset) {
throw new BadRequestException('Asset not found');
}
2022-12-05 11:56:44 -06:00
const updatedAsset = await this._assetRepository.update(authUser.id, asset, dto);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [assetId] } });
return mapAsset(updatedAsset);
}
async getAssetThumbnail(
authUser: AuthUserDto,
assetId: string,
query: GetAssetThumbnailDto,
res: Res,
headers: Record<string, string>,
) {
await this.access.requirePermission(authUser, Permission.ASSET_VIEW, assetId);
const asset = await this._assetRepository.get(assetId);
if (!asset) {
throw new NotFoundException('Asset not found');
}
try {
const [thumbnailPath, contentType] = this.getThumbnailPath(asset, query.format);
return this.streamFile(thumbnailPath, res, headers, contentType);
WIP refactor container and queuing system (#206) * refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
2022-06-11 16:12:06 -05:00
} catch (e) {
res.header('Cache-Control', 'none');
this.logger.error(`Cannot create read stream for asset ${asset.id}`, 'getAssetThumbnail');
throw new InternalServerErrorException(
`Cannot read thumbnail file for asset ${asset.id} - contact your administrator`,
{ cause: e as Error },
);
}
}
public async serveFile(
authUser: AuthUserDto,
assetId: string,
query: ServeFileDto,
res: Res,
headers: Record<string, string>,
) {
// this is not quite right as sometimes this returns the original still
await this.access.requirePermission(authUser, Permission.ASSET_VIEW, assetId);
const allowOriginalFile = !!(!authUser.isPublicUser || authUser.isAllowDownload);
const asset = await this._assetRepository.getById(assetId);
if (!asset) {
throw new NotFoundException('Asset does not exist');
}
// Handle Sending Images
if (asset.type == AssetType.IMAGE) {
try {
const { filepath, contentType } = this.getServePath(asset, query, allowOriginalFile);
return this.streamFile(filepath, res, headers, contentType);
WIP refactor container and queuing system (#206) * refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
2022-06-11 16:12:06 -05:00
} catch (e) {
this.logger.error(`Cannot create read stream for asset ${asset.id} ${JSON.stringify(e)}`, 'serveFile[IMAGE]');
throw new InternalServerErrorException(
e,
`Cannot read thumbnail file for asset ${asset.id} - contact your administrator`,
);
}
} else {
WIP refactor container and queuing system (#206) * refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
2022-06-11 16:12:06 -05:00
try {
const videoPath = asset.encodedVideoPath ? asset.encodedVideoPath : asset.originalPath;
const mimeType = asset.encodedVideoPath ? 'video/mp4' : asset.mimeType;
return this.streamFile(videoPath, res, headers, mimeType);
} catch (e: Error | any) {
this.logger.error(`Error serving VIDEO asset=${asset.id}`, e?.stack);
WIP refactor container and queuing system (#206) * refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app store
2022-06-11 16:12:06 -05:00
throw new InternalServerErrorException(`Failed to serve video asset ${e}`, 'ServeFile');
}
}
}
public async deleteAll(authUser: AuthUserDto, dto: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
const deleteQueue: Array<string | null> = [];
const result: DeleteAssetResponseDto[] = [];
const ids = dto.ids.slice();
for (const id of ids) {
const hasAccess = await this.access.hasPermission(authUser, Permission.ASSET_DELETE, id);
if (!hasAccess) {
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
continue;
}
const asset = await this._assetRepository.get(id);
if (!asset) {
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
continue;
}
try {
2023-05-17 13:07:17 -04:00
if (asset.faces) {
await Promise.all(
asset.faces.map(({ assetId, personId }) =>
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
),
);
}
await this._assetRepository.remove(asset);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
feat(server): support for read-only assets and importing existing items in the filesystem (#2715) * Added read-only flag for assets, endpoint to trigger file import vs upload * updated fixtures with new property * if upload is 'read-only', ensure there is no existing asset at the designated originalPath * added test for file import as well as detecting existing image at read-only destination location * Added storage service test for a case where it should not move read-only assets * upload doesn't need the read-only flag available, just importing * default isReadOnly on import endpoint to true * formatting fixes * create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation * updated code to reflect changes in MR * fixed read stream promise return type * new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates * refactor: import asset * chore: open api * chore: tests * Added externalPath support for individual users, updated UI to allow this to be set by admin * added missing var for externalPath in ui * chore: open api * fix: compilation issues * fix: server test * built api, fixed user-response dto to include externalPath * reverted accidental commit * bad commit of duplicate externalPath in user response dto * fixed tests to include externalPath on expected result * fix: unit tests * centralized supported filetypes, perform file type checking of asset and sidecar during file import process * centralized supported filetype check method to keep regex DRY * fixed typo * combined migrations into one * update api * Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not * update mimetype * Fixed detect correct mimetype * revert asset-upload config * reverted domain.constant * refactor * fix mime-type issue * fix format --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
if (!asset.isReadOnly) {
deleteQueue.push(
asset.originalPath,
asset.webpPath,
asset.resizePath,
asset.encodedVideoPath,
asset.sidecarPath,
);
}
// TODO refactor this to use cascades
if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
ids.push(asset.livePhotoVideoId);
}
} catch {
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
}
}
if (deleteQueue.length > 0) {
await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: deleteQueue } });
}
return result;
}
async getAssetSearchTerm(authUser: AuthUserDto): Promise<string[]> {
const possibleSearchTerm = new Set<string>();
const rows = await this._assetRepository.getSearchPropertiesByUserId(authUser.id);
rows.forEach((row: SearchPropertiesDto) => {
// tags
row.tags?.map((tag: string) => possibleSearchTerm.add(tag?.toLowerCase()));
// objects
row.objects?.map((object: string) => possibleSearchTerm.add(object?.toLowerCase()));
// asset's tyoe
possibleSearchTerm.add(row.assetType?.toLowerCase() || '');
// image orientation
possibleSearchTerm.add(row.orientation?.toLowerCase() || '');
// Lens model
possibleSearchTerm.add(row.lensModel?.toLowerCase() || '');
// Make and model
possibleSearchTerm.add(row.make?.toLowerCase() || '');
possibleSearchTerm.add(row.model?.toLowerCase() || '');
// Location
possibleSearchTerm.add(row.city?.toLowerCase() || '');
possibleSearchTerm.add(row.state?.toLowerCase() || '');
possibleSearchTerm.add(row.country?.toLowerCase() || '');
});
return Array.from(possibleSearchTerm).filter((x) => x != null && x != '');
}
2022-03-02 16:44:24 -06:00
async searchAsset(authUser: AuthUserDto, searchAssetDto: SearchAssetDto): Promise<AssetResponseDto[]> {
2022-03-02 16:44:24 -06:00
const query = `
SELECT a.*
FROM assets a
LEFT JOIN smart_info si ON a.id = si."assetId"
LEFT JOIN exif e ON a.id = e."assetId"
WHERE a."ownerId" = $1
AND
2022-03-02 16:44:24 -06:00
(
TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
e."exifTextSearchableColumn" @@ PLAINTO_TSQUERY('english', $2)
2022-03-02 16:44:24 -06:00
);
`;
const searchResults: AssetEntity[] = await this.assetRepository.query(query, [
authUser.id,
searchAssetDto.searchTerm,
]);
return searchResults.map((asset) => mapAsset(asset));
2022-03-02 16:44:24 -06:00
}
async getCuratedLocation(authUser: AuthUserDto): Promise<CuratedLocationsResponseDto[]> {
return this._assetRepository.getLocationsByUserId(authUser.id);
}
async getCuratedObject(authUser: AuthUserDto): Promise<CuratedObjectsResponseDto[]> {
return this._assetRepository.getDetectedObjectsByUserId(authUser.id);
}
async checkDuplicatedAsset(
authUser: AuthUserDto,
checkDuplicateAssetDto: CheckDuplicateAssetDto,
): Promise<CheckDuplicateAssetResponseDto> {
const res = await this.assetRepository.findOne({
where: {
deviceAssetId: checkDuplicateAssetDto.deviceAssetId,
deviceId: checkDuplicateAssetDto.deviceId,
ownerId: authUser.id,
},
});
const isDuplicated = res ? true : false;
return new CheckDuplicateAssetResponseDto(isDuplicated, res?.id);
}
async checkExistingAssets(
authUser: AuthUserDto,
checkExistingAssetsDto: CheckExistingAssetsDto,
): Promise<CheckExistingAssetsResponseDto> {
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
return {
existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto),
};
}
async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
// support base64 and hex checksums
for (const asset of dto.assets) {
if (asset.checksum.length === 28) {
asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
}
}
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
const checksumMap: Record<string, string> = {};
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
for (const { id, checksum } of results) {
checksumMap[checksum.toString('hex')] = id;
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
}
return {
results: dto.assets.map(({ id, checksum }) => {
const duplicate = checksumMap[checksum];
feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com>
2023-05-24 23:08:21 +02:00
if (duplicate) {
return {
id,
assetId: duplicate,
action: AssetUploadAction.REJECT,
reason: AssetRejectReason.DUPLICATE,
};
}
// TODO mime-check
return {
id,
action: AssetUploadAction.ACCEPT,
};
}),
};
}
async getAssetCountByTimeBucket(
authUser: AuthUserDto,
dto: GetAssetCountByTimeBucketDto,
): Promise<AssetCountByTimeBucketResponseDto> {
const userId = dto.userId || authUser.id;
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
const result = await this._assetRepository.getAssetCountByTimeBucket(userId, dto);
return mapAssetCountByTimeBucket(result);
}
getAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
return this._assetRepository.getAssetCountByUserId(authUser.id);
}
getArchivedAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
return this._assetRepository.getArchivedAssetCountByUserId(authUser.id);
}
getExifPermission(authUser: AuthUserDto) {
return !authUser.isPublicUser || authUser.isShowExif;
}
private getThumbnailPath(asset: AssetEntity, format: GetAssetThumbnailFormatEnum) {
switch (format) {
case GetAssetThumbnailFormatEnum.WEBP:
if (asset.webpPath) {
return [asset.webpPath, 'image/webp'];
}
this.logger.warn(`WebP thumbnail requested but not found for asset ${asset.id}, falling back to JPEG`);
case GetAssetThumbnailFormatEnum.JPEG:
default:
if (!asset.resizePath) {
throw new NotFoundException(`No thumbnail found for asset ${asset.id}`);
}
return [asset.resizePath, 'image/jpeg'];
}
}
private getServePath(asset: AssetEntity, query: ServeFileDto, allowOriginalFile: boolean): ServableFile {
/**
* Serve file viewer on the web
*/
if (query.isWeb && asset.mimeType != 'image/gif') {
if (!asset.resizePath) {
this.logger.error('Error serving IMAGE asset for web');
throw new InternalServerErrorException(`Failed to serve image asset for web`, 'ServeFile');
}
return { filepath: asset.resizePath, contentType: 'image/jpeg' };
}
/**
* Serve thumbnail image for both web and mobile app
*/
if ((!query.isThumb && allowOriginalFile) || (query.isWeb && asset.mimeType === 'image/gif')) {
return { filepath: asset.originalPath, contentType: asset.mimeType as string };
}
if (asset.webpPath && asset.webpPath.length > 0) {
return { filepath: asset.webpPath, contentType: 'image/webp' };
}
if (!asset.resizePath) {
throw new Error('resizePath not set');
}
return { filepath: asset.resizePath, contentType: 'image/jpeg' };
}
private async streamFile(filepath: string, res: Res, headers: Record<string, string>, contentType?: string | null) {
await fs.access(filepath, constants.R_OK);
const { size, mtimeNs } = await fs.stat(filepath, { bigint: true });
if (contentType) {
res.header('Content-Type', contentType);
}
const range = this.setResRange(res, headers, Number(size));
// etag
const etag = `W/"${size}-${mtimeNs}"`;
res.setHeader('ETag', etag);
if (etag === headers['if-none-match']) {
res.status(304);
return;
}
const stream = createReadStream(filepath, range);
return await pipeline(stream, res).catch((err) => {
if (err.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
this.logger.error(err);
}
});
}
private setResRange(res: Res, headers: Record<string, string>, size: number) {
if (!headers.range) {
return {};
}
/** Extracting Start and End value from Range Header */
const [startStr, endStr] = headers.range.replace(/bytes=/, '').split('-');
let start = parseInt(startStr, 10);
let end = endStr ? parseInt(endStr, 10) : size - 1;
if (!isNaN(start) && isNaN(end)) {
start = start;
end = size - 1;
}
if (isNaN(start) && !isNaN(end)) {
start = size - end;
end = size - 1;
}
// Handle unavailable range request
if (start >= size || end >= size) {
console.error('Bad Request');
res.status(416).set({ 'Content-Range': `bytes */${size}` });
throw new BadRequestException('Bad Request Range');
}
res.status(206).set({
'Content-Range': `bytes ${start}-${end}/${size}`,
'Accept-Ranges': 'bytes',
'Content-Length': end - start + 1,
});
return { start, end };
}
}