fix(maintenance): prevent enable/disable maintenance CLI hanging on occasion (#24713)

* fix(maintenance): prevent CLI hanging on occassion
fix(maintenance): always ack messages
fix(maintenance): ensure Redis is connected first

* chore(maintenance): validate app restart responses

* chore: mock the app restart callback

* fix: ack may not exist depending on caller

* refactor: move one shot into app.repository

* fix: send correct state in one shot

* chore: log restart event

server/src/services/cli.service.spec.ts

@@ -89,6 +89,7 @@ describe(CliService.name, () => {
         alreadyDisabled: true,
       });
 
+      expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0);
       expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
       expect(mocks.event.emit).toHaveBeenCalledTimes(0);
     });
@@ -99,6 +100,7 @@ describe(CliService.name, () => {
         alreadyDisabled: false,
       });
 
+      expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled();
       expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
         isMaintenanceMode: false,
       });
@@ -114,6 +116,7 @@ describe(CliService.name, () => {
         }),
       );
 
+      expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0);
       expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);
       expect(mocks.event.emit).toHaveBeenCalledTimes(0);
     });
@@ -126,6 +129,7 @@ describe(CliService.name, () => {
         }),
       );
 
+      expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled();
       expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {
         isMaintenanceMode: true,
         secret: expect.stringMatching(/^\w{128}$/),

server/src/services/cli.service.ts

@@ -5,7 +5,7 @@ import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
 import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';
 import { SystemMetadataKey } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
-import { createMaintenanceLoginUrl, generateMaintenanceSecret, sendOneShotAppRestart } from 'src/utils/maintenance';
+import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance';
 import { getExternalDomain } from 'src/utils/misc';
 
 @Injectable()
@@ -55,8 +55,7 @@ export class CliService extends BaseService {
 
     const state = { isMaintenanceMode: false as const };
     await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);
-
-    sendOneShotAppRestart(state);
+    await this.appRepository.sendOneShotAppRestart(state);
 
     return {
       alreadyDisabled: false,
@@ -89,7 +88,7 @@ export class CliService extends BaseService {
       secret,
     });
 
-    sendOneShotAppRestart({
+    await this.appRepository.sendOneShotAppRestart({
       isMaintenanceMode: true,
     });
 

server/src/services/maintenance.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { OnEvent } from 'src/decorators';
 import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';
 import { SystemMetadataKey } from 'src/enum';
+import { ArgOf } from 'src/repositories/event.repository';
 import { BaseService } from 'src/services/base.service';
 import { MaintenanceModeState } from 'src/types';
 import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance';
@@ -31,7 +32,10 @@ export class MaintenanceService extends BaseService {
   }
 
   @OnEvent({ name: 'AppRestart', server: true })
-  onRestart(): void {
+  onRestart(event: ArgOf<'AppRestart'>, ack?: (ok: 'ok') => void): void {
+    this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`);
+
+    ack?.('ok');
     this.appRepository.exitApp();
   }