diff --git a/server/src/maintenance/maintenance-websocket.repository.ts b/server/src/maintenance/maintenance-websocket.repository.ts index 5d8368cf69..cf04c0ad12 100644 --- a/server/src/maintenance/maintenance-websocket.repository.ts +++ b/server/src/maintenance/maintenance-websocket.repository.ts @@ -37,7 +37,13 @@ export class MaintenanceWebsocketRepository implements OnGatewayConnection, OnGa afterInit(websocketServer: Server) { this.logger.log('Initialized websocket server'); - websocketServer.on('AppRestart', () => this.appRepository.exitApp()); + + websocketServer.on('AppRestart', (event: ArgsOf<'AppRestart'>, ack?: (ok: 'ok') => void) => { + this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`); + + ack?.('ok'); + this.appRepository.exitApp(); + }); } clientBroadcast(event: T, ...data: ClientEventMap[T]) { diff --git a/server/src/repositories/app.repository.ts b/server/src/repositories/app.repository.ts index e6181ef7f3..96e413232f 100644 --- a/server/src/repositories/app.repository.ts +++ b/server/src/repositories/app.repository.ts @@ -1,5 +1,10 @@ import { Injectable } from '@nestjs/common'; +import { createAdapter } from '@socket.io/redis-adapter'; +import Redis from 'ioredis'; +import { Server as SocketIO } from 'socket.io'; import { ExitCode } from 'src/enum'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { AppRestartEvent } from 'src/repositories/event.repository'; @Injectable() export class AppRepository { @@ -17,4 +22,26 @@ export class AppRepository { setCloseFn(fn: () => Promise) { this.closeFn = fn; } + + async sendOneShotAppRestart(state: AppRestartEvent): Promise { + const server = new SocketIO(); + const { redis } = new ConfigRepository().getEnv(); + const pubClient = new Redis({ ...redis, lazyConnect: true }); + const subClient = pubClient.duplicate(); + + await Promise.all([pubClient.connect(), subClient.connect()]); + + server.adapter(createAdapter(pubClient, subClient)); + + // => corresponds to notification.service.ts#onAppRestart + server.emit('AppRestartV1', state, async () => { + const responses = await server.serverSideEmitWithAck('AppRestart', state); + if (responses.some((response) => response !== 'ok')) { + throw new Error("One or more node(s) returned a non-'ok' response to our restart request!"); + } + + pubClient.disconnect(); + subClient.disconnect(); + }); + } } diff --git a/server/src/services/cli.service.spec.ts b/server/src/services/cli.service.spec.ts index 49fa5cf5b8..f4f14c3e68 100644 --- a/server/src/services/cli.service.spec.ts +++ b/server/src/services/cli.service.spec.ts @@ -89,6 +89,7 @@ describe(CliService.name, () => { alreadyDisabled: true, }); + expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0); expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0); expect(mocks.event.emit).toHaveBeenCalledTimes(0); }); @@ -99,6 +100,7 @@ describe(CliService.name, () => { alreadyDisabled: false, }); + expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled(); expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, { isMaintenanceMode: false, }); @@ -114,6 +116,7 @@ describe(CliService.name, () => { }), ); + expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0); expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0); expect(mocks.event.emit).toHaveBeenCalledTimes(0); }); @@ -126,6 +129,7 @@ describe(CliService.name, () => { }), ); + expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled(); expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, { isMaintenanceMode: true, secret: expect.stringMatching(/^\w{128}$/), diff --git a/server/src/services/cli.service.ts b/server/src/services/cli.service.ts index 3d248edc7a..8d2f1b0e99 100644 --- a/server/src/services/cli.service.ts +++ b/server/src/services/cli.service.ts @@ -5,7 +5,7 @@ import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto'; import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto'; import { SystemMetadataKey } from 'src/enum'; import { BaseService } from 'src/services/base.service'; -import { createMaintenanceLoginUrl, generateMaintenanceSecret, sendOneShotAppRestart } from 'src/utils/maintenance'; +import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance'; import { getExternalDomain } from 'src/utils/misc'; @Injectable() @@ -55,8 +55,7 @@ export class CliService extends BaseService { const state = { isMaintenanceMode: false as const }; await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state); - - sendOneShotAppRestart(state); + await this.appRepository.sendOneShotAppRestart(state); return { alreadyDisabled: false, @@ -89,7 +88,7 @@ export class CliService extends BaseService { secret, }); - sendOneShotAppRestart({ + await this.appRepository.sendOneShotAppRestart({ isMaintenanceMode: true, }); diff --git a/server/src/services/maintenance.service.ts b/server/src/services/maintenance.service.ts index e6808300bc..0f5fa06957 100644 --- a/server/src/services/maintenance.service.ts +++ b/server/src/services/maintenance.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { OnEvent } from 'src/decorators'; import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto'; import { SystemMetadataKey } from 'src/enum'; +import { ArgOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { MaintenanceModeState } from 'src/types'; import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance'; @@ -31,7 +32,10 @@ export class MaintenanceService extends BaseService { } @OnEvent({ name: 'AppRestart', server: true }) - onRestart(): void { + onRestart(event: ArgOf<'AppRestart'>, ack?: (ok: 'ok') => void): void { + this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`); + + ack?.('ok'); this.appRepository.exitApp(); } diff --git a/server/src/utils/maintenance.ts b/server/src/utils/maintenance.ts index 22de2e4083..faa92395d6 100644 --- a/server/src/utils/maintenance.ts +++ b/server/src/utils/maintenance.ts @@ -1,55 +1,6 @@ -import { createAdapter } from '@socket.io/redis-adapter'; -import Redis from 'ioredis'; import { SignJWT } from 'jose'; import { randomBytes } from 'node:crypto'; -import { Server as SocketIO } from 'socket.io'; import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto'; -import { ConfigRepository } from 'src/repositories/config.repository'; -import { AppRestartEvent } from 'src/repositories/event.repository'; - -export function sendOneShotAppRestart(state: AppRestartEvent): void { - const server = new SocketIO(); - const { redis } = new ConfigRepository().getEnv(); - const pubClient = new Redis(redis); - const subClient = pubClient.duplicate(); - server.adapter(createAdapter(pubClient, subClient)); - - /** - * Keep trying until we manage to stop Immich - * - * Sometimes there appear to be communication - * issues between to the other servers. - * - * This issue only occurs with this method. - */ - async function tryTerminate() { - while (true) { - try { - const responses = await server.serverSideEmitWithAck('AppRestart', state); - if (responses.length > 0) { - return; - } - } catch (error) { - console.error(error); - console.error('Encountered an error while telling Immich to stop.'); - } - - console.info( - "\nIt doesn't appear that Immich stopped, trying again in a moment.\nIf Immich is already not running, you can ignore this error.", - ); - - await new Promise((r) => setTimeout(r, 1e3)); - } - } - - // => corresponds to notification.service.ts#onAppRestart - server.emit('AppRestartV1', state, () => { - void tryTerminate().finally(() => { - pubClient.disconnect(); - subClient.disconnect(); - }); - }); -} export async function createMaintenanceLoginUrl( baseUrl: string,