2023-06-16 15:54:17 -04:00
|
|
|
import {
|
2023-06-28 09:56:24 -04:00
|
|
|
AccessCore,
|
2023-06-16 15:54:17 -04:00
|
|
|
AssetResponseDto,
|
2023-07-09 00:37:40 -04:00
|
|
|
ASSET_MIME_TYPES,
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
AuthUserDto,
|
2023-06-16 15:54:17 -04:00
|
|
|
getLivePhotoMotionFilename,
|
|
|
|
|
IAccessRepository,
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
ICryptoRepository,
|
2023-06-16 15:54:17 -04:00
|
|
|
IJobRepository,
|
|
|
|
|
IStorageRepository,
|
|
|
|
|
JobName,
|
2023-07-09 00:37:40 -04:00
|
|
|
LIVE_PHOTO_MIME_TYPES,
|
2023-06-16 15:54:17 -04:00
|
|
|
mapAsset,
|
|
|
|
|
mapAssetWithoutExif,
|
2023-06-28 09:56:24 -04:00
|
|
|
Permission,
|
2023-07-09 00:37:40 -04:00
|
|
|
PROFILE_MIME_TYPES,
|
|
|
|
|
SIDECAR_MIME_TYPES,
|
|
|
|
|
StorageCore,
|
|
|
|
|
StorageFolder,
|
|
|
|
|
UploadFieldName,
|
|
|
|
|
UploadFile,
|
2023-06-16 15:54:17 -04:00
|
|
|
} from '@app/domain';
|
2023-06-20 21:08:43 -04:00
|
|
|
import { AssetEntity, AssetType } from '@app/infra/entities';
|
2022-06-25 19:53:06 +02:00
|
|
|
import {
|
|
|
|
|
BadRequestException,
|
2022-08-26 22:53:37 -07:00
|
|
|
Inject,
|
2022-06-25 19:53:06 +02:00
|
|
|
Injectable,
|
|
|
|
|
InternalServerErrorException,
|
|
|
|
|
Logger,
|
|
|
|
|
NotFoundException,
|
|
|
|
|
} from '@nestjs/common';
|
2022-02-03 10:06:44 -06:00
|
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
2022-02-13 15:10:42 -06:00
|
|
|
import { Response as Res } from 'express';
|
2023-07-02 22:37:12 -04:00
|
|
|
import { constants, createReadStream } from 'fs';
|
2022-07-01 12:00:12 -05:00
|
|
|
import fs from 'fs/promises';
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
import mime from 'mime-types';
|
2023-07-09 00:37:40 -04:00
|
|
|
import path, { extname } from 'path';
|
|
|
|
|
import sanitize from 'sanitize-filename';
|
2023-07-02 22:37:12 -04:00
|
|
|
import { pipeline } from 'stream/promises';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { QueryFailedError, Repository } from 'typeorm';
|
2023-07-09 00:37:40 -04:00
|
|
|
import { UploadRequest } from '../../app.interceptor';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { IAssetRepository } from './asset-repository';
|
|
|
|
|
import { AssetCore } from './asset.core';
|
|
|
|
|
import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
|
|
|
|
|
import { AssetSearchDto } from './dto/asset-search.dto';
|
2022-07-06 16:12:55 -05:00
|
|
|
import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
|
2023-07-09 00:37:40 -04:00
|
|
|
import { CreateAssetDto, ImportAssetDto } from './dto/create-asset.dto';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { DeleteAssetDto } from './dto/delete-asset.dto';
|
|
|
|
|
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
|
|
|
|
|
import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
|
2022-07-15 23:18:17 -05:00
|
|
|
import { GetAssetThumbnailDto, GetAssetThumbnailFormatEnum } from './dto/get-asset-thumbnail.dto';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { SearchAssetDto } from './dto/search-asset.dto';
|
2022-08-26 22:53:37 -07:00
|
|
|
import { SearchPropertiesDto } from './dto/search-properties.dto';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { ServeFileDto } from './dto/serve-file.dto';
|
|
|
|
|
import { UpdateAssetDto } from './dto/update-asset.dto';
|
|
|
|
|
import {
|
|
|
|
|
AssetBulkUploadCheckResponseDto,
|
|
|
|
|
AssetRejectReason,
|
|
|
|
|
AssetUploadAction,
|
|
|
|
|
} from './response-dto/asset-check-response.dto';
|
2022-08-26 22:53:37 -07:00
|
|
|
import {
|
2022-09-04 08:34:39 -05:00
|
|
|
AssetCountByTimeBucketResponseDto,
|
|
|
|
|
mapAssetCountByTimeBucket,
|
2022-08-26 22:53:37 -07:00
|
|
|
} from './response-dto/asset-count-by-time-group-response.dto';
|
2022-09-07 15:16:18 -05:00
|
|
|
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
|
2022-11-18 23:12:54 -06:00
|
|
|
import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
|
2023-06-16 15:54:17 -04:00
|
|
|
import { CheckDuplicateAssetResponseDto } from './response-dto/check-duplicate-asset-response.dto';
|
|
|
|
|
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
|
|
|
|
|
import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
|
|
|
|
|
import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
|
|
|
|
|
import { DeleteAssetResponseDto, DeleteAssetStatusEnum } from './response-dto/delete-asset-response.dto';
|
2022-02-13 15:10:42 -06:00
|
|
|
|
2023-03-23 22:40:30 -04:00
|
|
|
interface ServableFile {
|
|
|
|
|
filepath: string;
|
|
|
|
|
contentType: string;
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-03 10:06:44 -06:00
|
|
|
@Injectable()
|
|
|
|
|
export class AssetService {
|
2023-01-09 14:16:08 -06:00
|
|
|
readonly logger = new Logger(AssetService.name);
|
2023-01-30 11:14:13 -05:00
|
|
|
private assetCore: AssetCore;
|
2023-06-28 09:56:24 -04:00
|
|
|
private access: AccessCore;
|
2023-07-09 00:37:40 -04:00
|
|
|
private storageCore = new StorageCore();
|
2023-01-09 14:16:08 -06:00
|
|
|
|
2022-02-03 10:06:44 -06:00
|
|
|
constructor(
|
2023-06-28 09:56:24 -04:00
|
|
|
@Inject(IAccessRepository) accessRepository: IAccessRepository,
|
2022-12-30 08:22:06 -05:00
|
|
|
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
|
2023-06-20 21:08:43 -04:00
|
|
|
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
|
2023-01-21 23:13:36 -05:00
|
|
|
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
2023-02-25 09:12:03 -05:00
|
|
|
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
2023-01-09 14:16:08 -06:00
|
|
|
) {
|
2023-03-28 16:04:11 -04:00
|
|
|
this.assetCore = new AssetCore(_assetRepository, jobRepository);
|
2023-06-28 09:56:24 -04:00
|
|
|
this.access = new AccessCore(accessRepository);
|
2023-01-09 14:16:08 -06:00
|
|
|
}
|
2022-06-11 16:12:06 -05:00
|
|
|
|
2023-07-09 00:37:40 -04:00
|
|
|
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {
|
|
|
|
|
this.access.requireUploadAccess(authUser);
|
|
|
|
|
|
|
|
|
|
switch (fieldName) {
|
|
|
|
|
case UploadFieldName.ASSET_DATA:
|
|
|
|
|
if (ASSET_MIME_TYPES.includes(file.mimeType)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case UploadFieldName.LIVE_PHOTO_DATA:
|
|
|
|
|
if (LIVE_PHOTO_MIME_TYPES.includes(file.mimeType)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case UploadFieldName.SIDECAR_DATA:
|
|
|
|
|
if (SIDECAR_MIME_TYPES.includes(file.mimeType)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case UploadFieldName.PROFILE_DATA:
|
|
|
|
|
if (PROFILE_MIME_TYPES.includes(file.mimeType)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const ext = extname(file.originalName);
|
|
|
|
|
this.logger.error(`Unsupported file type ${ext} file MIME type ${file.mimeType}`);
|
|
|
|
|
throw new BadRequestException(`Unsupported file type ${ext}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
getUploadFilename({ authUser, fieldName, file }: UploadRequest): string {
|
|
|
|
|
this.access.requireUploadAccess(authUser);
|
|
|
|
|
|
|
|
|
|
const originalExt = extname(file.originalName);
|
|
|
|
|
|
|
|
|
|
const lookup = {
|
|
|
|
|
[UploadFieldName.ASSET_DATA]: originalExt,
|
|
|
|
|
[UploadFieldName.LIVE_PHOTO_DATA]: '.mov',
|
|
|
|
|
[UploadFieldName.SIDECAR_DATA]: '.xmp',
|
|
|
|
|
[UploadFieldName.PROFILE_DATA]: originalExt,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
getUploadFolder({ authUser, fieldName }: UploadRequest): string {
|
|
|
|
|
authUser = this.access.requireUploadAccess(authUser);
|
|
|
|
|
|
|
|
|
|
let folder = this.storageCore.getFolderLocation(StorageFolder.UPLOAD, authUser.id);
|
|
|
|
|
if (fieldName === UploadFieldName.PROFILE_DATA) {
|
|
|
|
|
folder = this.storageCore.getFolderLocation(StorageFolder.PROFILE, authUser.id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.storageRepository.mkdirSync(folder);
|
|
|
|
|
|
|
|
|
|
return folder;
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
public async uploadFile(
|
2022-11-18 23:12:54 -06:00
|
|
|
authUser: AuthUserDto,
|
2023-01-30 11:14:13 -05:00
|
|
|
dto: CreateAssetDto,
|
|
|
|
|
file: UploadFile,
|
|
|
|
|
livePhotoFile?: UploadFile,
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-24 21:59:30 -04:00
|
|
|
sidecarFile?: UploadFile,
|
2023-01-30 11:14:13 -05:00
|
|
|
): Promise<AssetFileUploadResponseDto> {
|
|
|
|
|
if (livePhotoFile) {
|
2023-02-25 09:12:03 -05:00
|
|
|
livePhotoFile = {
|
|
|
|
|
...livePhotoFile,
|
2023-03-28 16:04:11 -04:00
|
|
|
originalName: getLivePhotoMotionFilename(file.originalName, livePhotoFile.originalName),
|
2023-02-25 09:12:03 -05:00
|
|
|
};
|
2023-01-30 11:14:13 -05:00
|
|
|
}
|
2022-12-16 14:26:12 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
let livePhotoAsset: AssetEntity | null = null;
|
2022-11-18 23:12:54 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
try {
|
|
|
|
|
if (livePhotoFile) {
|
|
|
|
|
const livePhotoDto = { ...dto, assetType: AssetType.VIDEO, isVisible: false };
|
|
|
|
|
livePhotoAsset = await this.assetCore.create(authUser, livePhotoDto, livePhotoFile);
|
2022-11-18 23:12:54 -06:00
|
|
|
}
|
|
|
|
|
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
const asset = await this.assetCore.create(authUser, dto, file, livePhotoAsset?.id, sidecarFile?.originalPath);
|
2022-12-16 14:26:12 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
return { id: asset.id, duplicate: false };
|
|
|
|
|
} catch (error: any) {
|
|
|
|
|
// clean up files
|
2023-02-25 09:12:03 -05:00
|
|
|
await this.jobRepository.queue({
|
|
|
|
|
name: JobName.DELETE_FILES,
|
feat(server): xmp sidecar metadata (#2466)
* initial commit for XMP sidecar support
* Added support for 'missing' metadata files to include those without sidecar files, now detects sidecar files in the filesystem for media already ingested but the sidecar was created afterwards
* didn't mean to commit default log level during testing
* new sidecar logic for video metadata as well
* Added xml mimetype for sidecars only
* don't need capture group for this regex
* wrong default value reverted
* simplified the move here - keep it in the same try catch since the outcome is to move the media back anyway
* simplified setter logic
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
* simplified logic per suggestions
* sidecar is now its own queue with a discover and sync, updated UI for the new job queueing
* queue a sidecar job for every asset based on discovery or sync, though the logic is almost identical aside from linking the sidecar
* now queue sidecar jobs for each assset, though logic is mostly the same between discovery and sync
* simplified logic of filename extraction and asset instantiation
* not sure how that got deleted..
* updated code per suggestions and comments in the PR
* stat was not being used, removed the variable set
* better type checking, using in-scope variables for exif getter instead of passing in every time
* removed commented out test
* ran and resolved all lints, formats, checks, and tests
* resolved suggested change in PR
* made getExifProperty more dynamic with multiple possible args for fallbacks, fixed typo, used generic in function for better type checking
* better error handling and moving files back to positions on move or save failure
* regenerated api
* format fixes
* Added XMP documentation
* documentation typo
* Merged in main
* missed merge conflict
* more changes due to a merge
* Resolving conflicts
* added icon for sidecar jobs
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-05-24 21:59:30 -04:00
|
|
|
data: { files: [file.originalPath, livePhotoFile?.originalPath, sidecarFile?.originalPath] },
|
2023-01-21 23:13:36 -05:00
|
|
|
});
|
2022-11-18 23:12:54 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
// handle duplicates with a success response
|
|
|
|
|
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
|
2023-05-24 23:08:21 +02:00
|
|
|
const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum);
|
|
|
|
|
const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
|
2023-01-30 11:14:13 -05:00
|
|
|
return { id: duplicate.id, duplicate: true };
|
2022-11-18 23:12:54 -06:00
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
this.logger.error(`Error uploading file ${error}`, error?.stack);
|
|
|
|
|
throw new BadRequestException(`Error uploading file`, `${error}`);
|
2022-11-18 23:12:54 -06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
public async importFile(authUser: AuthUserDto, dto: ImportAssetDto): Promise<AssetFileUploadResponseDto> {
|
|
|
|
|
dto = {
|
|
|
|
|
...dto,
|
|
|
|
|
assetPath: path.resolve(dto.assetPath),
|
|
|
|
|
sidecarPath: dto.sidecarPath ? path.resolve(dto.sidecarPath) : undefined,
|
|
|
|
|
};
|
|
|
|
|
|
2023-07-09 00:37:40 -04:00
|
|
|
const mimeType = mime.lookup(dto.assetPath) as string;
|
|
|
|
|
if (!ASSET_MIME_TYPES.includes(mimeType)) {
|
|
|
|
|
throw new BadRequestException(`Unsupported file type ${mimeType}`);
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (dto.sidecarPath) {
|
2023-06-28 10:40:21 -04:00
|
|
|
if (path.extname(dto.sidecarPath).toLowerCase() !== '.xmp') {
|
|
|
|
|
throw new BadRequestException(`Unsupported sidecar file type`);
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const filepath of [dto.assetPath, dto.sidecarPath]) {
|
|
|
|
|
if (!filepath) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 12:25:08 -04:00
|
|
|
const exists = await this.storageRepository.checkFileExists(filepath, constants.R_OK);
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
if (!exists) {
|
|
|
|
|
throw new BadRequestException('File does not exist');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!authUser.externalPath || !dto.assetPath.match(new RegExp(`^${authUser.externalPath}`))) {
|
|
|
|
|
throw new BadRequestException("File does not exist within user's external path");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const assetFile: UploadFile = {
|
|
|
|
|
checksum: await this.cryptoRepository.hashFile(dto.assetPath),
|
2023-07-09 00:37:40 -04:00
|
|
|
mimeType,
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
originalPath: dto.assetPath,
|
|
|
|
|
originalName: path.parse(dto.assetPath).name,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const asset = await this.assetCore.create(authUser, dto, assetFile, undefined, dto.sidecarPath);
|
|
|
|
|
return { id: asset.id, duplicate: false };
|
|
|
|
|
} catch (error: QueryFailedError | Error | any) {
|
|
|
|
|
// handle duplicates with a success response
|
|
|
|
|
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
|
|
|
|
|
const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, [assetFile.checksum]);
|
|
|
|
|
return { id: duplicate.id, duplicate: true };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_4ed4f8052685ff5b1e7ca1058ba') {
|
|
|
|
|
const duplicate = await this._assetRepository.getByOriginalPath(dto.assetPath);
|
|
|
|
|
if (duplicate) {
|
|
|
|
|
if (duplicate.ownerId === authUser.id) {
|
|
|
|
|
return { id: duplicate.id, duplicate: true };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new BadRequestException('Path in use by another user');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.logger.error(`Error importing file ${error}`, error?.stack);
|
|
|
|
|
throw new BadRequestException(`Error importing file`, `${error}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-02-03 10:06:44 -06:00
|
|
|
public async getUserAssetsByDeviceId(authUser: AuthUserDto, deviceId: string) {
|
2022-08-26 22:53:37 -07:00
|
|
|
return this._assetRepository.getAllByDeviceId(authUser.id, deviceId);
|
2022-02-03 10:06:44 -06:00
|
|
|
}
|
|
|
|
|
|
2023-01-23 23:16:20 -05:00
|
|
|
public async getAllAssets(authUser: AuthUserDto, dto: AssetSearchDto): Promise<AssetResponseDto[]> {
|
2023-06-28 09:56:24 -04:00
|
|
|
const userId = dto.userId || authUser.id;
|
|
|
|
|
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
|
|
|
|
|
const assets = await this._assetRepository.getAllByUserId(userId, dto);
|
2022-07-08 21:26:50 -05:00
|
|
|
return assets.map((asset) => mapAsset(asset));
|
2022-02-13 15:10:42 -06:00
|
|
|
}
|
|
|
|
|
|
2023-06-28 09:56:24 -04:00
|
|
|
public async getAssetByTimeBucket(authUser: AuthUserDto, dto: GetAssetByTimeBucketDto): Promise<AssetResponseDto[]> {
|
|
|
|
|
const userId = dto.userId || authUser.id;
|
|
|
|
|
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
|
|
|
|
|
const assets = await this._assetRepository.getAssetByTimeBucket(userId, dto);
|
2022-09-04 08:34:39 -05:00
|
|
|
return assets.map((asset) => mapAsset(asset));
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-21 22:15:16 -06:00
|
|
|
public async getAssetById(authUser: AuthUserDto, assetId: string): Promise<AssetResponseDto> {
|
2023-06-28 09:56:24 -04:00
|
|
|
await this.access.requirePermission(authUser, Permission.ASSET_READ, assetId);
|
2023-06-06 15:17:15 -04:00
|
|
|
|
2023-01-21 22:15:16 -06:00
|
|
|
const allowExif = this.getExifPermission(authUser);
|
2022-08-26 22:53:37 -07:00
|
|
|
const asset = await this._assetRepository.getById(assetId);
|
2022-07-08 21:26:50 -05:00
|
|
|
|
2023-01-21 22:15:16 -06:00
|
|
|
if (allowExif) {
|
|
|
|
|
return mapAsset(asset);
|
|
|
|
|
} else {
|
|
|
|
|
return mapAssetWithoutExif(asset);
|
|
|
|
|
}
|
2022-02-10 20:40:11 -06:00
|
|
|
}
|
2022-02-13 15:10:42 -06:00
|
|
|
|
2022-12-05 11:56:44 -06:00
|
|
|
public async updateAsset(authUser: AuthUserDto, assetId: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
|
2023-06-28 09:56:24 -04:00
|
|
|
await this.access.requirePermission(authUser, Permission.ASSET_UPDATE, assetId);
|
2023-06-06 15:17:15 -04:00
|
|
|
|
2022-11-08 11:20:36 -05:00
|
|
|
const asset = await this._assetRepository.getById(assetId);
|
|
|
|
|
if (!asset) {
|
|
|
|
|
throw new BadRequestException('Asset not found');
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-05 11:56:44 -06:00
|
|
|
const updatedAsset = await this._assetRepository.update(authUser.id, asset, dto);
|
2022-11-08 11:20:36 -05:00
|
|
|
|
2023-03-18 08:44:42 -05:00
|
|
|
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [assetId] } });
|
2023-03-02 21:47:08 -05:00
|
|
|
|
2022-11-08 11:20:36 -05:00
|
|
|
return mapAsset(updatedAsset);
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-06 15:17:15 -04:00
|
|
|
async getAssetThumbnail(
|
|
|
|
|
authUser: AuthUserDto,
|
|
|
|
|
assetId: string,
|
|
|
|
|
query: GetAssetThumbnailDto,
|
|
|
|
|
res: Res,
|
|
|
|
|
headers: Record<string, string>,
|
|
|
|
|
) {
|
2023-06-28 09:56:24 -04:00
|
|
|
await this.access.requirePermission(authUser, Permission.ASSET_VIEW, assetId);
|
|
|
|
|
|
2023-02-03 10:16:25 -05:00
|
|
|
const asset = await this._assetRepository.get(assetId);
|
2022-07-01 12:00:12 -05:00
|
|
|
if (!asset) {
|
|
|
|
|
throw new NotFoundException('Asset not found');
|
|
|
|
|
}
|
2022-04-23 21:08:45 -05:00
|
|
|
|
2022-07-01 12:00:12 -05:00
|
|
|
try {
|
2023-07-08 16:07:56 -04:00
|
|
|
const [thumbnailPath, contentType] = this.getThumbnailPath(asset, query.format);
|
|
|
|
|
return this.streamFile(thumbnailPath, res, headers, contentType);
|
2022-06-11 16:12:06 -05:00
|
|
|
} catch (e) {
|
2022-09-05 00:18:53 -05:00
|
|
|
res.header('Cache-Control', 'none');
|
2023-07-02 22:37:12 -04:00
|
|
|
this.logger.error(`Cannot create read stream for asset ${asset.id}`, 'getAssetThumbnail');
|
2022-07-01 12:00:12 -05:00
|
|
|
throw new InternalServerErrorException(
|
|
|
|
|
`Cannot read thumbnail file for asset ${asset.id} - contact your administrator`,
|
2023-03-18 08:44:42 -05:00
|
|
|
{ cause: e as Error },
|
2022-07-01 12:00:12 -05:00
|
|
|
);
|
2022-05-22 06:56:36 -05:00
|
|
|
}
|
2022-04-23 21:08:45 -05:00
|
|
|
}
|
|
|
|
|
|
2023-01-21 22:15:16 -06:00
|
|
|
public async serveFile(
|
|
|
|
|
authUser: AuthUserDto,
|
|
|
|
|
assetId: string,
|
|
|
|
|
query: ServeFileDto,
|
|
|
|
|
res: Res,
|
|
|
|
|
headers: Record<string, string>,
|
|
|
|
|
) {
|
2023-06-28 09:56:24 -04:00
|
|
|
// this is not quite right as sometimes this returns the original still
|
|
|
|
|
await this.access.requirePermission(authUser, Permission.ASSET_VIEW, assetId);
|
2023-06-06 15:17:15 -04:00
|
|
|
|
2023-03-23 22:40:30 -04:00
|
|
|
const allowOriginalFile = !!(!authUser.isPublicUser || authUser.isAllowDownload);
|
2023-01-21 22:15:16 -06:00
|
|
|
|
2022-11-16 00:11:16 -06:00
|
|
|
const asset = await this._assetRepository.getById(assetId);
|
2022-04-02 12:31:53 -05:00
|
|
|
if (!asset) {
|
2022-07-01 12:00:12 -05:00
|
|
|
throw new NotFoundException('Asset does not exist');
|
2022-04-02 12:31:53 -05:00
|
|
|
}
|
2022-05-27 14:02:06 -05:00
|
|
|
|
2022-02-13 15:10:42 -06:00
|
|
|
// Handle Sending Images
|
2022-07-10 21:41:45 -05:00
|
|
|
if (asset.type == AssetType.IMAGE) {
|
2022-07-01 12:00:12 -05:00
|
|
|
try {
|
2023-03-23 22:40:30 -04:00
|
|
|
const { filepath, contentType } = this.getServePath(asset, query, allowOriginalFile);
|
|
|
|
|
return this.streamFile(filepath, res, headers, contentType);
|
2022-06-11 16:12:06 -05:00
|
|
|
} catch (e) {
|
2023-07-02 22:37:12 -04:00
|
|
|
this.logger.error(`Cannot create read stream for asset ${asset.id} ${JSON.stringify(e)}`, 'serveFile[IMAGE]');
|
2022-07-01 12:00:12 -05:00
|
|
|
throw new InternalServerErrorException(
|
|
|
|
|
e,
|
|
|
|
|
`Cannot read thumbnail file for asset ${asset.id} - contact your administrator`,
|
|
|
|
|
);
|
2022-06-04 18:34:11 -05:00
|
|
|
}
|
2022-07-10 21:41:45 -05:00
|
|
|
} else {
|
2022-06-11 16:12:06 -05:00
|
|
|
try {
|
2023-07-02 22:37:12 -04:00
|
|
|
const videoPath = asset.encodedVideoPath ? asset.encodedVideoPath : asset.originalPath;
|
|
|
|
|
const mimeType = asset.encodedVideoPath ? 'video/mp4' : asset.mimeType;
|
2023-03-23 22:40:30 -04:00
|
|
|
|
2023-03-29 12:36:18 -05:00
|
|
|
return this.streamFile(videoPath, res, headers, mimeType);
|
2023-06-30 12:25:08 -04:00
|
|
|
} catch (e: Error | any) {
|
|
|
|
|
this.logger.error(`Error serving VIDEO asset=${asset.id}`, e?.stack);
|
2022-06-11 16:12:06 -05:00
|
|
|
throw new InternalServerErrorException(`Failed to serve video asset ${e}`, 'ServeFile');
|
2022-02-13 15:10:42 -06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
public async deleteAll(authUser: AuthUserDto, dto: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
|
2023-02-25 09:12:03 -05:00
|
|
|
const deleteQueue: Array<string | null> = [];
|
2022-07-13 07:23:48 -05:00
|
|
|
const result: DeleteAssetResponseDto[] = [];
|
2022-02-13 15:10:42 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
const ids = dto.ids.slice();
|
|
|
|
|
for (const id of ids) {
|
2023-06-28 09:56:24 -04:00
|
|
|
const hasAccess = await this.access.hasPermission(authUser, Permission.ASSET_DELETE, id);
|
|
|
|
|
if (!hasAccess) {
|
|
|
|
|
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
const asset = await this._assetRepository.get(id);
|
|
|
|
|
if (!asset) {
|
|
|
|
|
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2022-02-13 15:10:42 -06:00
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
try {
|
2023-05-17 13:07:17 -04:00
|
|
|
if (asset.faces) {
|
|
|
|
|
await Promise.all(
|
|
|
|
|
asset.faces.map(({ assetId, personId }) =>
|
|
|
|
|
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
|
|
|
|
|
),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
await this._assetRepository.remove(asset);
|
2023-03-18 08:44:42 -05:00
|
|
|
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
|
2023-01-30 11:14:13 -05:00
|
|
|
|
2023-02-02 22:37:39 -06:00
|
|
|
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });
|
feat(server): support for read-only assets and importing existing items in the filesystem (#2715)
* Added read-only flag for assets, endpoint to trigger file import vs upload
* updated fixtures with new property
* if upload is 'read-only', ensure there is no existing asset at the designated originalPath
* added test for file import as well as detecting existing image at read-only destination location
* Added storage service test for a case where it should not move read-only assets
* upload doesn't need the read-only flag available, just importing
* default isReadOnly on import endpoint to true
* formatting fixes
* create-asset dto needs isReadOnly, so set it to false by default on create, updated api generation
* updated code to reflect changes in MR
* fixed read stream promise return type
* new index for originalPath, check for existing path on import, reglardless of user, to prevent duplicates
* refactor: import asset
* chore: open api
* chore: tests
* Added externalPath support for individual users, updated UI to allow this to be set by admin
* added missing var for externalPath in ui
* chore: open api
* fix: compilation issues
* fix: server test
* built api, fixed user-response dto to include externalPath
* reverted accidental commit
* bad commit of duplicate externalPath in user response dto
* fixed tests to include externalPath on expected result
* fix: unit tests
* centralized supported filetypes, perform file type checking of asset and sidecar during file import process
* centralized supported filetype check method to keep regex DRY
* fixed typo
* combined migrations into one
* update api
* Removed externalPath from shared-link code, added column to admin user page whether external paths / import is enabled or not
* update mimetype
* Fixed detect correct mimetype
* revert asset-upload config
* reverted domain.constant
* refactor
* fix mime-type issue
* fix format
---------
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2023-06-21 22:33:20 -04:00
|
|
|
|
|
|
|
|
if (!asset.isReadOnly) {
|
|
|
|
|
deleteQueue.push(
|
|
|
|
|
asset.originalPath,
|
|
|
|
|
asset.webpPath,
|
|
|
|
|
asset.resizePath,
|
|
|
|
|
asset.encodedVideoPath,
|
|
|
|
|
asset.sidecarPath,
|
|
|
|
|
);
|
|
|
|
|
}
|
2023-01-30 11:14:13 -05:00
|
|
|
|
|
|
|
|
// TODO refactor this to use cascades
|
|
|
|
|
if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
|
|
|
|
|
ids.push(asset.livePhotoVideoId);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
result.push({ id, status: DeleteAssetStatusEnum.FAILED });
|
2022-02-13 15:10:42 -06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-30 11:14:13 -05:00
|
|
|
if (deleteQueue.length > 0) {
|
2023-02-25 09:12:03 -05:00
|
|
|
await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: deleteQueue } });
|
2023-01-30 11:14:13 -05:00
|
|
|
}
|
|
|
|
|
|
2022-02-13 15:10:42 -06:00
|
|
|
return result;
|
|
|
|
|
}
|
2022-02-27 12:43:29 -06:00
|
|
|
|
2022-03-27 14:58:54 -05:00
|
|
|
async getAssetSearchTerm(authUser: AuthUserDto): Promise<string[]> {
|
|
|
|
|
const possibleSearchTerm = new Set<string>();
|
2022-02-27 12:43:29 -06:00
|
|
|
|
2022-08-26 22:53:37 -07:00
|
|
|
const rows = await this._assetRepository.getSearchPropertiesByUserId(authUser.id);
|
|
|
|
|
rows.forEach((row: SearchPropertiesDto) => {
|
2022-02-27 12:43:29 -06:00
|
|
|
// tags
|
2022-08-26 22:53:37 -07:00
|
|
|
row.tags?.map((tag: string) => possibleSearchTerm.add(tag?.toLowerCase()));
|
2022-02-27 12:43:29 -06:00
|
|
|
|
2022-03-27 14:58:54 -05:00
|
|
|
// objects
|
2022-08-26 22:53:37 -07:00
|
|
|
row.objects?.map((object: string) => possibleSearchTerm.add(object?.toLowerCase()));
|
2022-03-27 14:58:54 -05:00
|
|
|
|
2022-02-27 12:43:29 -06:00
|
|
|
// asset's tyoe
|
2022-08-26 22:53:37 -07:00
|
|
|
possibleSearchTerm.add(row.assetType?.toLowerCase() || '');
|
2022-02-27 12:43:29 -06:00
|
|
|
|
|
|
|
|
// image orientation
|
2022-08-26 22:53:37 -07:00
|
|
|
possibleSearchTerm.add(row.orientation?.toLowerCase() || '');
|
2022-02-27 12:43:29 -06:00
|
|
|
|
|
|
|
|
// Lens model
|
2022-08-26 22:53:37 -07:00
|
|
|
possibleSearchTerm.add(row.lensModel?.toLowerCase() || '');
|
2022-02-27 12:43:29 -06:00
|
|
|
|
|
|
|
|
// Make and model
|
2022-08-26 22:53:37 -07:00
|
|
|
possibleSearchTerm.add(row.make?.toLowerCase() || '');
|
|
|
|
|
possibleSearchTerm.add(row.model?.toLowerCase() || '');
|
2022-03-10 16:09:03 -06:00
|
|
|
|
|
|
|
|
// Location
|
2022-08-26 22:53:37 -07:00
|
|
|
possibleSearchTerm.add(row.city?.toLowerCase() || '');
|
|
|
|
|
possibleSearchTerm.add(row.state?.toLowerCase() || '');
|
|
|
|
|
possibleSearchTerm.add(row.country?.toLowerCase() || '');
|
2022-02-27 12:43:29 -06:00
|
|
|
});
|
|
|
|
|
|
2022-08-26 22:53:37 -07:00
|
|
|
return Array.from(possibleSearchTerm).filter((x) => x != null && x != '');
|
2022-02-27 12:43:29 -06:00
|
|
|
}
|
2022-03-02 16:44:24 -06:00
|
|
|
|
2022-07-08 21:26:50 -05:00
|
|
|
async searchAsset(authUser: AuthUserDto, searchAssetDto: SearchAssetDto): Promise<AssetResponseDto[]> {
|
2022-03-02 16:44:24 -06:00
|
|
|
const query = `
|
|
|
|
|
SELECT a.*
|
|
|
|
|
FROM assets a
|
|
|
|
|
LEFT JOIN smart_info si ON a.id = si."assetId"
|
|
|
|
|
LEFT JOIN exif e ON a.id = e."assetId"
|
|
|
|
|
|
2023-02-20 16:58:46 +01:00
|
|
|
WHERE a."ownerId" = $1
|
2022-06-25 19:53:06 +02:00
|
|
|
AND
|
2022-03-02 16:44:24 -06:00
|
|
|
(
|
2022-03-27 14:58:54 -05:00
|
|
|
TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
|
|
|
|
|
TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
|
2022-07-04 20:20:43 +01:00
|
|
|
e."exifTextSearchableColumn" @@ PLAINTO_TSQUERY('english', $2)
|
2022-03-02 16:44:24 -06:00
|
|
|
);
|
|
|
|
|
`;
|
|
|
|
|
|
2022-07-08 21:26:50 -05:00
|
|
|
const searchResults: AssetEntity[] = await this.assetRepository.query(query, [
|
|
|
|
|
authUser.id,
|
|
|
|
|
searchAssetDto.searchTerm,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
return searchResults.map((asset) => mapAsset(asset));
|
2022-03-02 16:44:24 -06:00
|
|
|
}
|
2022-03-16 10:19:31 -05:00
|
|
|
|
2022-08-26 22:53:37 -07:00
|
|
|
async getCuratedLocation(authUser: AuthUserDto): Promise<CuratedLocationsResponseDto[]> {
|
|
|
|
|
return this._assetRepository.getLocationsByUserId(authUser.id);
|
2022-03-27 14:58:54 -05:00
|
|
|
}
|
2022-03-16 10:19:31 -05:00
|
|
|
|
2022-07-08 21:26:50 -05:00
|
|
|
async getCuratedObject(authUser: AuthUserDto): Promise<CuratedObjectsResponseDto[]> {
|
2022-08-26 22:53:37 -07:00
|
|
|
return this._assetRepository.getDetectedObjectsByUserId(authUser.id);
|
2022-03-16 10:19:31 -05:00
|
|
|
}
|
2022-06-19 08:16:35 -05:00
|
|
|
|
2022-07-26 20:53:25 -05:00
|
|
|
async checkDuplicatedAsset(
|
|
|
|
|
authUser: AuthUserDto,
|
|
|
|
|
checkDuplicateAssetDto: CheckDuplicateAssetDto,
|
|
|
|
|
): Promise<CheckDuplicateAssetResponseDto> {
|
2022-06-19 08:16:35 -05:00
|
|
|
const res = await this.assetRepository.findOne({
|
|
|
|
|
where: {
|
2022-07-06 16:12:55 -05:00
|
|
|
deviceAssetId: checkDuplicateAssetDto.deviceAssetId,
|
|
|
|
|
deviceId: checkDuplicateAssetDto.deviceId,
|
2023-02-19 16:44:53 +00:00
|
|
|
ownerId: authUser.id,
|
2022-06-19 08:16:35 -05:00
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
2022-07-26 20:53:25 -05:00
|
|
|
const isDuplicated = res ? true : false;
|
|
|
|
|
|
|
|
|
|
return new CheckDuplicateAssetResponseDto(isDuplicated, res?.id);
|
2022-06-19 08:16:35 -05:00
|
|
|
}
|
2022-08-26 22:53:37 -07:00
|
|
|
|
2022-10-25 09:51:03 -05:00
|
|
|
async checkExistingAssets(
|
|
|
|
|
authUser: AuthUserDto,
|
|
|
|
|
checkExistingAssetsDto: CheckExistingAssetsDto,
|
|
|
|
|
): Promise<CheckExistingAssetsResponseDto> {
|
2023-05-24 23:08:21 +02:00
|
|
|
return {
|
|
|
|
|
existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
|
2023-05-27 21:56:17 -04:00
|
|
|
// support base64 and hex checksums
|
|
|
|
|
for (const asset of dto.assets) {
|
|
|
|
|
if (asset.checksum.length === 28) {
|
|
|
|
|
asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-24 23:08:21 +02:00
|
|
|
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
|
|
|
|
|
const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
|
2023-05-27 21:56:17 -04:00
|
|
|
const checksumMap: Record<string, string> = {};
|
2023-05-24 23:08:21 +02:00
|
|
|
|
|
|
|
|
for (const { id, checksum } of results) {
|
2023-05-27 21:56:17 -04:00
|
|
|
checksumMap[checksum.toString('hex')] = id;
|
2023-05-24 23:08:21 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
results: dto.assets.map(({ id, checksum }) => {
|
2023-05-27 21:56:17 -04:00
|
|
|
const duplicate = checksumMap[checksum];
|
2023-05-24 23:08:21 +02:00
|
|
|
if (duplicate) {
|
|
|
|
|
return {
|
|
|
|
|
id,
|
|
|
|
|
assetId: duplicate,
|
|
|
|
|
action: AssetUploadAction.REJECT,
|
|
|
|
|
reason: AssetRejectReason.DUPLICATE,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TODO mime-check
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
id,
|
|
|
|
|
action: AssetUploadAction.ACCEPT,
|
|
|
|
|
};
|
|
|
|
|
}),
|
|
|
|
|
};
|
2022-10-25 09:51:03 -05:00
|
|
|
}
|
|
|
|
|
|
2022-09-04 08:34:39 -05:00
|
|
|
async getAssetCountByTimeBucket(
|
2022-08-26 22:53:37 -07:00
|
|
|
authUser: AuthUserDto,
|
2023-06-28 09:56:24 -04:00
|
|
|
dto: GetAssetCountByTimeBucketDto,
|
2022-09-04 08:34:39 -05:00
|
|
|
): Promise<AssetCountByTimeBucketResponseDto> {
|
2023-06-28 09:56:24 -04:00
|
|
|
const userId = dto.userId || authUser.id;
|
|
|
|
|
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
|
|
|
|
|
const result = await this._assetRepository.getAssetCountByTimeBucket(userId, dto);
|
2022-09-04 08:34:39 -05:00
|
|
|
return mapAssetCountByTimeBucket(result);
|
2022-08-26 22:53:37 -07:00
|
|
|
}
|
2022-08-31 21:27:17 +07:00
|
|
|
|
2022-09-07 15:16:18 -05:00
|
|
|
getAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
|
|
|
|
|
return this._assetRepository.getAssetCountByUserId(authUser.id);
|
|
|
|
|
}
|
2022-12-04 18:42:36 +01:00
|
|
|
|
2023-04-12 18:37:52 +03:00
|
|
|
getArchivedAssetCountByUserId(authUser: AuthUserDto): Promise<AssetCountByUserIdResponseDto> {
|
|
|
|
|
return this._assetRepository.getArchivedAssetCountByUserId(authUser.id);
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-21 22:15:16 -06:00
|
|
|
getExifPermission(authUser: AuthUserDto) {
|
|
|
|
|
return !authUser.isPublicUser || authUser.isShowExif;
|
2023-01-14 23:49:47 -06:00
|
|
|
}
|
2022-11-29 22:45:47 +01:00
|
|
|
|
2023-03-23 22:40:30 -04:00
|
|
|
private getThumbnailPath(asset: AssetEntity, format: GetAssetThumbnailFormatEnum) {
|
|
|
|
|
switch (format) {
|
|
|
|
|
case GetAssetThumbnailFormatEnum.WEBP:
|
2023-07-08 16:07:56 -04:00
|
|
|
if (asset.webpPath) {
|
|
|
|
|
return [asset.webpPath, 'image/webp'];
|
2023-03-23 22:40:30 -04:00
|
|
|
}
|
2023-07-08 16:07:56 -04:00
|
|
|
this.logger.warn(`WebP thumbnail requested but not found for asset ${asset.id}, falling back to JPEG`);
|
2023-03-23 22:40:30 -04:00
|
|
|
|
|
|
|
|
case GetAssetThumbnailFormatEnum.JPEG:
|
|
|
|
|
default:
|
|
|
|
|
if (!asset.resizePath) {
|
2023-07-08 16:07:56 -04:00
|
|
|
throw new NotFoundException(`No thumbnail found for asset ${asset.id}`);
|
2023-03-23 22:40:30 -04:00
|
|
|
}
|
2023-07-08 16:07:56 -04:00
|
|
|
return [asset.resizePath, 'image/jpeg'];
|
2023-03-23 22:40:30 -04:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private getServePath(asset: AssetEntity, query: ServeFileDto, allowOriginalFile: boolean): ServableFile {
|
|
|
|
|
/**
|
|
|
|
|
* Serve file viewer on the web
|
|
|
|
|
*/
|
|
|
|
|
if (query.isWeb && asset.mimeType != 'image/gif') {
|
|
|
|
|
if (!asset.resizePath) {
|
|
|
|
|
this.logger.error('Error serving IMAGE asset for web');
|
|
|
|
|
throw new InternalServerErrorException(`Failed to serve image asset for web`, 'ServeFile');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { filepath: asset.resizePath, contentType: 'image/jpeg' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Serve thumbnail image for both web and mobile app
|
|
|
|
|
*/
|
|
|
|
|
if ((!query.isThumb && allowOriginalFile) || (query.isWeb && asset.mimeType === 'image/gif')) {
|
|
|
|
|
return { filepath: asset.originalPath, contentType: asset.mimeType as string };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (asset.webpPath && asset.webpPath.length > 0) {
|
|
|
|
|
return { filepath: asset.webpPath, contentType: 'image/webp' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!asset.resizePath) {
|
|
|
|
|
throw new Error('resizePath not set');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { filepath: asset.resizePath, contentType: 'image/jpeg' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async streamFile(filepath: string, res: Res, headers: Record<string, string>, contentType?: string | null) {
|
2023-07-02 22:37:12 -04:00
|
|
|
await fs.access(filepath, constants.R_OK);
|
|
|
|
|
const { size, mtimeNs } = await fs.stat(filepath, { bigint: true });
|
|
|
|
|
|
2023-03-23 22:40:30 -04:00
|
|
|
if (contentType) {
|
|
|
|
|
res.header('Content-Type', contentType);
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-02 22:37:12 -04:00
|
|
|
const range = this.setResRange(res, headers, Number(size));
|
|
|
|
|
|
2023-03-23 22:40:30 -04:00
|
|
|
// etag
|
|
|
|
|
const etag = `W/"${size}-${mtimeNs}"`;
|
|
|
|
|
res.setHeader('ETag', etag);
|
|
|
|
|
if (etag === headers['if-none-match']) {
|
|
|
|
|
res.status(304);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-02 22:37:12 -04:00
|
|
|
const stream = createReadStream(filepath, range);
|
|
|
|
|
return await pipeline(stream, res).catch((err) => {
|
|
|
|
|
if (err.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
|
|
|
|
|
this.logger.error(err);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private setResRange(res: Res, headers: Record<string, string>, size: number) {
|
|
|
|
|
if (!headers.range) {
|
|
|
|
|
return {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Extracting Start and End value from Range Header */
|
|
|
|
|
const [startStr, endStr] = headers.range.replace(/bytes=/, '').split('-');
|
|
|
|
|
let start = parseInt(startStr, 10);
|
|
|
|
|
let end = endStr ? parseInt(endStr, 10) : size - 1;
|
|
|
|
|
|
|
|
|
|
if (!isNaN(start) && isNaN(end)) {
|
|
|
|
|
start = start;
|
|
|
|
|
end = size - 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (isNaN(start) && !isNaN(end)) {
|
|
|
|
|
start = size - end;
|
|
|
|
|
end = size - 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Handle unavailable range request
|
|
|
|
|
if (start >= size || end >= size) {
|
|
|
|
|
console.error('Bad Request');
|
|
|
|
|
res.status(416).set({ 'Content-Range': `bytes */${size}` });
|
|
|
|
|
|
|
|
|
|
throw new BadRequestException('Bad Request Range');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
res.status(206).set({
|
|
|
|
|
'Content-Range': `bytes ${start}-${end}/${size}`,
|
|
|
|
|
'Accept-Ranges': 'bytes',
|
|
|
|
|
'Content-Length': end - start + 1,
|
|
|
|
|
});
|
2023-03-23 22:40:30 -04:00
|
|
|
|
2023-07-02 22:37:12 -04:00
|
|
|
return { start, end };
|
2022-11-29 22:45:47 +01:00
|
|
|
}
|
|
|
|
|
}
|