immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-29 01:11:52 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

feat(cli): refactor and add album support (#4434)

Browse Source 
* Allow building and installing cli

* feat: add format fix

* docs: remove cli folder

* feat: use immich scoped package

* feat: rewrite cli readme

* docs: add info on running without building

* cleanup

* chore: remove import functionality from cli

* feat: add logout to cli

* docs: add todo for file format from server

* docs: add compilation step to cli

* fix: success message spacing

* feat: can create albums

* fix: add check step to cli

* fix: typos

* feat: pull file formats from server

* chore: use crawl service from server

* chore: fix lint

* docs: add cli documentation

* chore: rename ignore pattern

* chore: add version number to cli

* feat: use sdk

* fix: cleanup

* feat: album name on windows

* chore: remove skipped asset field

* feat: add more info to server-info command

* chore: cleanup

* chore: remove unneeded packages

* chore: fix docs links

* feat: add cli v2 milestone

* fix: set correct cli date

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors
2023-11-19 23:16:24 +01:00
committed by GitHub
parent 88b5f5b500
commit 7e38e7c949
31 changed files with 637 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

421
cli/src/services/crawl.service.spec.ts
View File

@@ -1,235 +1,206 @@
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-unused-vars */
import { CrawlService } from './crawl.service';
import mockfs from 'mock-fs';
import { toIncludeSameMembers } from 'jest-extended';
import { CrawlOptionsDto } from '../cores/dto/crawl-options-dto';
import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto';
import { CrawlService } from '.';
const matchers = require('jest-extended');
expect.extend(matchers);
interface Test {
test: string;
options: CrawlOptionsDto;
files: Record<string, boolean>;
}
const crawlService = new CrawlService();
const cwd = process.cwd();
describe('CrawlService', () => {
beforeAll(() => {
// Write a dummy output before mock-fs to prevent some annoying errors
console.log();
});
const tests: Test[] = [
{
test: 'should return empty when crawling an empty path list',
options: {
pathsToCrawl: [],
},
files: {},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should exclude by file extension',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.tif'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by file extension without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.TIF'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by folder',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/raw/**'],
},
files: {
'/photos/image.jpg': true,
'/photos/raw/image.jpg': false,
'/photos/raw2/image.jpg': true,
'/photos/folder/raw/image.jpg': false,
'/photos/crawl/image.jpg': true,
},
},
{
test: 'should crawl multiple paths',
options: {
pathsToCrawl: ['/photos/', '/images/', '/albums/'],
},
files: {
'/photos/image1.jpg': true,
'/images/image2.jpg': true,
'/albums/image3.jpg': true,
},
},
{
test: 'should support globbing paths',
options: {
pathsToCrawl: ['/photos*'],
},
files: {
'/photos1/image1.jpg': true,
'/photos2/image2.jpg': true,
'/images/image3.jpg': false,
},
},
{
test: 'should crawl a single path without trailing slash',
options: {
pathsToCrawl: ['/photos'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/subfolder/image1.jpg': true,
'/photos/subfolder/image2.jpg': true,
'/image1.jpg': false,
},
},
{
test: 'should filter file extensions',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.txt': false,
'/photos/1': false,
},
},
{
test: 'should include photo and video extensions',
options: {
pathsToCrawl: ['/photos/', '/videos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.jpeg': true,
'/photos/image.heic': true,
'/photos/image.heif': true,
'/photos/image.png': true,
'/photos/image.gif': true,
'/photos/image.tif': true,
'/photos/image.tiff': true,
'/photos/image.webp': true,
'/photos/image.dng': true,
'/photos/image.nef': true,
'/videos/video.mp4': true,
'/videos/video.mov': true,
'/videos/video.webm': true,
},
},
{
test: 'should check file extensions without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.Jpg': true,
'/photos/image.jpG': true,
'/photos/image.JPG': true,
'/photos/image.jpEg': true,
'/photos/image.TIFF': true,
'/photos/image.tif': true,
'/photos/image.dng': true,
'/photos/image.NEF': true,
},
},
{
test: 'should normalize the path',
options: {
pathsToCrawl: ['/photos/1/../2'],
},
files: {
'/photos/1/image.jpg': false,
'/photos/2/image.jpg': true,
},
},
{
test: 'should return absolute paths',
options: {
pathsToCrawl: ['photos'],
},
files: {
[`${cwd}/photos/1.jpg`]: true,
[`${cwd}/photos/2.jpg`]: true,
[`/photos/3.jpg`]: false,
},
},
];
it('should crawl a single directory', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single file', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/image.jpg'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a file and a directory', async () => {
mockfs({
'/photos/image.jpg': '',
'/images/photo.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/image.jpg', '/images/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/images/photo.jpg']);
});
it('should exclude by file extension', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.tif': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/*.tif'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should exclude by file extension without case sensitivity', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.tif': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/*.TIF'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should exclude by folder', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/raw/image.jpg': '',
'/photos/raw2/image.jpg': '',
'/photos/folder/raw/image.jpg': '',
'/photos/crawl/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/raw/**'];
options.recursive = true;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/photos/raw2/image.jpg', '/photos/crawl/image.jpg']);
});
it('should crawl multiple paths', async () => {
mockfs({
'/photos/image1.jpg': '',
'/images/image2.jpg': '',
'/albums/image3.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/', '/images/', '/albums/'];
options.recursive = false;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image1.jpg', '/images/image2.jpg', '/albums/image3.jpg']);
});
it('should crawl a single path without trailing slash', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single path without recursion', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/subfolder/image1.jpg': '',
'/photos/subfolder/image2.jpg': '',
'/image1.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single path with recursion', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/subfolder/image1.jpg': '',
'/photos/subfolder/image2.jpg': '',
'/image1.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.recursive = true;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/subfolder/image1.jpg',
'/photos/subfolder/image2.jpg',
]);
});
it('should filter file extensions', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.txt': '',
'/photos/1': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should include photo and video extensions', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.jpeg': '',
'/photos/image.heic': '',
'/photos/image.heif': '',
'/photos/image.png': '',
'/photos/image.gif': '',
'/photos/image.tif': '',
'/photos/image.tiff': '',
'/photos/image.webp': '',
'/photos/image.dng': '',
'/photos/image.nef': '',
'/videos/video.mp4': '',
'/videos/video.mov': '',
'/videos/video.webm': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/', '/videos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/image.jpeg',
'/photos/image.heic',
'/photos/image.heif',
'/photos/image.png',
'/photos/image.gif',
'/photos/image.tif',
'/photos/image.tiff',
'/photos/image.webp',
'/photos/image.dng',
'/photos/image.nef',
'/videos/video.mp4',
'/videos/video.mov',
'/videos/video.webm',
]);
});
it('should check file extensions without case sensitivity', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.Jpg': '',
'/photos/image.jpG': '',
'/photos/image.JPG': '',
'/photos/image.jpEg': '',
'/photos/image.TIFF': '',
'/photos/image.tif': '',
'/photos/image.dng': '',
'/photos/image.NEF': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/image.Jpg',
'/photos/image.jpG',
'/photos/image.JPG',
'/photos/image.jpEg',
'/photos/image.TIFF',
'/photos/image.tif',
'/photos/image.dng',
'/photos/image.NEF',
]);
});
describe(CrawlService.name, () => {
const sut = new CrawlService(
['.jpg', '.jpeg', '.png', '.heif', '.heic', '.tif', '.nef', '.webp', '.tiff', '.dng', '.gif'],
['.mov', '.mp4', '.webm'],
);
afterEach(() => {
mockfs.restore();
});
describe('crawl', () => {
for (const { test, options, files } of tests) {
it(test, async () => {
mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, ''])));
const actual = await sut.crawl(options);
const expected = Object.entries(files)
.filter((entry) => entry[1])
.map(([file]) => file);
expect(actual.sort()).toEqual(expected.sort());
});
}
});
});

47
cli/src/services/crawl.service.ts
View File

@@ -1,47 +1,28 @@
import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto';
import { ACCEPTED_FILE_EXTENSIONS } from '../cores';
import { glob } from 'glob';
import * as fs from 'fs';
export class CrawlService {
public async crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
const pathsToCrawl: string[] = crawlOptions.pathsToCrawl;
private readonly extensions!: string[];
const directories: string[] = [];
const crawledFiles: string[] = [];
constructor(image: string[], video: string[]) {
this.extensions = image.concat(video).map((extension) => extension.replace('.', ''));
}
for await (const currentPath of pathsToCrawl) {
const stats = await fs.promises.stat(currentPath);
if (stats.isFile() || stats.isSymbolicLink()) {
crawledFiles.push(currentPath);
} else {
directories.push(currentPath);
}
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
if (!pathsToCrawl) {
return Promise.resolve([]);
}
let searchPattern: string;
if (directories.length === 1) {
searchPattern = directories[0];
} else if (directories.length === 0) {
return crawledFiles;
} else {
searchPattern = '{' + directories.join(',') + '}';
}
const base = pathsToCrawl.length === 1 ? pathsToCrawl[0] : `{${pathsToCrawl.join(',')}}`;
const extensions = `*{${this.extensions}}`;
if (crawlOptions.recursive) {
searchPattern = searchPattern + '/**/';
}
searchPattern = `${searchPattern}/*.{${ACCEPTED_FILE_EXTENSIONS.join(',')}}`;
const globbedFiles = await glob(searchPattern, {
return glob(`${base}/**/${extensions}`, {
absolute: true,
nocase: true,
nodir: true,
ignore: crawlOptions.excludePatterns,
dot: includeHidden,
ignore: exclusionPatterns,
});
const returnedFiles = crawledFiles.concat(globbedFiles);
returnedFiles.sort();
return returnedFiles;
}
}

1
cli/src/services/index.ts
View File

@@ -1,2 +1 @@
export * from './upload.service';
export * from './crawl.service';

4
cli/src/services/session.service.ts
View File

@@ -46,7 +46,7 @@ export class SessionService {
// Check if server and api key are valid
const { data: userInfo } = await this.api.userApi.getMyUserInfo().catch((error) => {
throw new LoginError(`Failed to connect to the server: ${error.message}`);
throw new LoginError(`Failed to connect to server ${instanceUrl}: ${error.message}`);
});
console.log(`Logged in as ${userInfo.email}`);
@@ -78,7 +78,7 @@ export class SessionService {
private async ping(): Promise<void> {
const { data: pingResponse } = await this.api.serverInfoApi.pingServer().catch((error) => {
throw new Error(`Failed to connect to the server: ${error.message}`);
throw new Error(`Failed to connect to server ${this.api.apiConfiguration.instanceUrl}: ${error.message}`);
});
if (pingResponse.res !== 'pong') {

24
cli/src/services/upload.service.spec.ts
View File

@@ -1,24 +0,0 @@
import { UploadService } from './upload.service';
import axios from 'axios';
import FormData from 'form-data';
import { ApiConfiguration } from '../cores/api-configuration';
jest.mock('axios', () => jest.fn());
describe('UploadService', () => {
let uploadService: UploadService;
beforeEach(() => {
const apiConfiguration = new ApiConfiguration('https://example.com/api', 'key');
uploadService = new UploadService(apiConfiguration);
});
it('should call axios', async () => {
const data = new FormData();
await uploadService.upload(data);
expect(axios).toHaveBeenCalled();
});
});

65
cli/src/services/upload.service.ts
View File

@@ -1,65 +0,0 @@
import axios, { AxiosRequestConfig } from 'axios';
import FormData from 'form-data';
import { ApiConfiguration } from '../cores/api-configuration';
export class UploadService {
private readonly uploadConfig: AxiosRequestConfig<any>;
private readonly checkAssetExistenceConfig: AxiosRequestConfig<any>;
private readonly importConfig: AxiosRequestConfig<any>;
constructor(apiConfiguration: ApiConfiguration) {
this.uploadConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/upload`,
headers: {
'x-api-key': apiConfiguration.apiKey,
},
maxContentLength: Number.POSITIVE_INFINITY,
maxBodyLength: Number.POSITIVE_INFINITY,
};
this.importConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/import`,
headers: {
'x-api-key': apiConfiguration.apiKey,
'Content-Type': 'application/json',
},
maxContentLength: Number.POSITIVE_INFINITY,
maxBodyLength: Number.POSITIVE_INFINITY,
};
this.checkAssetExistenceConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/bulk-upload-check`,
headers: {
'x-api-key': apiConfiguration.apiKey,
'Content-Type': 'application/json',
},
};
}
public checkIfAssetAlreadyExists(path: string, checksum: string) {
this.checkAssetExistenceConfig.data = JSON.stringify({ assets: [{ id: path, checksum: checksum }] });
// TODO: retry on 500 errors?
return axios(this.checkAssetExistenceConfig);
}
public upload(data: FormData) {
this.uploadConfig.data = data;
// TODO: retry on 500 errors?
return axios(this.uploadConfig);
}
public import(data: any) {
this.importConfig.data = data;
// TODO: retry on 500 errors?
return axios(this.importConfig);
}
}