feat: restore database backups (#23978)

* feat: ProcessRepository#createSpawnDuplexStream

* test: write tests for ProcessRepository#createSpawnDuplexStream

* feat: StorageRepository#createGzip,createGunzip,createPlainReadStream

* feat: backups util (args, create, restore, progress)

* feat: wait on maintenance operation lock on boot

* chore: use backup util from backup.service.ts
test: update backup.service.ts tests with new util

* feat: list/delete backups (maintenance services)

* chore: open api
fix: missing action in cli.service.ts

* chore: add missing repositories to MaintenanceModule

* refactor: move logSecret into module init

* feat: initialise StorageCore in maintenance mode

* feat: authenticate websocket requests in maintenance mode

* test: add mock for new storage fns

* feat: add MaintenanceEphemeralStateRepository
refactor: cache the secret in memory

* test: update service worker tests

* feat: add external maintenance mode status

* feat: synchronised status, restore db action

* test: backup restore service tests

* refactor: DRY end maintenance

* feat: list and delete backup routes

* feat: start action on boot

* fix: should set status on restore end

* refactor: add maintenanceStore to hold writables

* feat: sync status to web app

* feat: web impl.

* test: various utils for testings

* test: web e2e tests

* test: e2e maintenance spec

* test: update cli spec

* chore: e2e lint

* chore: lint fixes

* chore: lint fixes

* feat: start restore flow route

* test: update e2e tests

* chore: remove neon lights on maintenance action pages

* fix: use 'startRestoreFlow' on onboarding page

* chore: ignore any library folder in `docker/`

* fix: load status on boot

* feat: upload backups

* refactor: permit any .sql(.gz) to be listed/restored

* feat: download backups from list

* fix: permit uploading just .sql files

* feat: restore just .sql files

* fix: don't show backups list if logged out

* feat: system integrity check in restore flow

* test: not providing failed backups in API anymore

* test: util should also not try to use failedBackups

* fix: actually assign inputStream

* test: correct test backup prep.

* fix: ensure task is defined to show error

* test: fix docker cp command

* test: update e2e web spec to select next button

* test: update e2e api tests

* test: refactor timeouts

* chore: remove `showDelete` from maint. settings

* chore: lint

* chore: lint

* fix: make sure backups are correctly sorted for clean up

* test: update service spec

* test: adjust e2e timeout

* test: increase web timeouts for ci

* chore: move gitignore changes

* chore: additional filename validation

* refactor: better typings for integrity API

* feat: higher accuracy progress tracking

* chore: delay lock retry

* refactor: remove old maintenance settings

* refactor: clean up tailwind classes

* refactor: use while loop rather than recursive calls

* test: update service specs

* chore: check canParse too

* chore: lint

* fix: logic error causing infinite loop

* refactor: use <ProgressBar /> from ui library

* fix: create or overwrite file

* chore: i18n pass, update progress bar

* fix: wrong translation string

* chore: update colour variables

* test: update web test for new maint. page

* chore: format, fix key

* test: update tests to be more linter complaint & use new routines

* chore: update onClick -> onAction, title -> breadcrumbs

* fix: use wrench icon in admin settings sidebar

* chore: add translation strings to accordion

* chore: lint

* refactor: move maintenance worker init into service

* refactor: `maintenanceStatus` -> `getMaintenanceStatus`
refactor: `integrityCheck` -> `detectPriorInstall`
chore: add `v2.4.0` version
refactor: `/backups/list` -> `/backups`
refactor: use sendFile in download route
refactor: use separate backups permissions
chore: correct descriptions
refactor: permit handler that doesn't return promise for sendfile

* refactor: move status impl into service
refactor: add active flag to maintenance status

* refactor: split into database backup controller

* test: split api e2e tests and passing

* fix: move end button into authed default maint page

* fix: also show in restore flow

* fix: import getMaintenanceStatus

* test: split web e2e tests

* refactor: ensure detect install is consistently named

* chore: ensure admin for detect install while out of maint.

* refactor: remove state repository

* test: update maint. worker service spec

* test: split backup service spec

* refactor: rename db backup routes

* refactor: instead of param, allow bulk backup deletion

* test: update sdk use in e2e test

* test: correct deleteBackup call

* fix: correct type for serverinstall response dto

* chore: validate filename for deletion

* test: wip

* test: backups no longer take path param

* refactor: scope util to database-backups instead of backups

* fix: update worker controller with new route

* chore: use new admin page actions

* chore: remove stray comment

* test: rename outdated test

* refactor: getter pattern for maintenance secret

* refactor: `createSpawnDuplexStream` -> `spawnDuplexStream`

* refactor: prefer `Object.assign`

* refactor: remove useless try {} block

* refactor: prefer `type Props`
refactor: prefer arrow function

* refactor: use luxon API for minutesAgo

* chore: remove change to gitignore

* refactor: prefer `type Props`

* refactor: remove async from onMount

* refactor: use luxon toRelative for relative time

* refactor: duplicate logic check

* chore: open api

* refactor: begin moving code into web//services

* refactor: don't use template string with $t

* test: use dialog role to match prompt

* refactor: split actions into flow/restore

* test: fix action value

* refactor: move more service calls into web//services

* chore: should void fn return

* chore: bump 2.4.0 to 2.5.0 in controller

* chore: bump 2.4.0 to 2.5.0 in controller

* refactor: use events for web//services

* chore: open api

* chore: open api

* refactor: don't await returned promise

* refactor: remove redundant check

* refactor: add `type: command` to actions

* refactor: split backup entries into own component

* refactor: split restore flow into separate components

* refactor(web): split BackupDelete event

* chore: stylings

* chore: stylings

* fix: don't log query failure on first boot

* feat: support pg_dumpall backups

* feat: display information about each backup

* chore: i18n

* feat: rollback to restore point on migrations failure

* feat: health check after restore

* chore: format

* refactor: split health check into separate function

* refactor: split health into repository
test: write tests covering rollbacks

* fix: omit 'health' requirement from createDbBackup

* test(e2e): rollback test

* fix: wrap text in backup entry

* fix: don't shrink context menu button

* fix: correct CREATE DB syntax for postgres

* test: rename backups generated by test

* feat: add filesize to backup response dto

* feat: restore list

* feat: ui work

* fix: e2e test

* fix: e2e test

* pr feedback

* pr feedback

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>

e2e/src/utils.ts

@@ -6,7 +6,9 @@ import {
   CheckExistingAssetsDto,
   CreateAlbumDto,
   CreateLibraryDto,
+  JobCreateDto,
   MaintenanceAction,
+  ManualJobName,
   MetadataSearchDto,
   Permission,
   PersonCreateDto,
@@ -21,6 +23,7 @@ import {
   checkExistingAssets,
   createAlbum,
   createApiKey,
+  createJob,
   createLibrary,
   createPartner,
   createPerson,
@@ -28,10 +31,12 @@ import {
   createStack,
   createUserAdmin,
   deleteAssets,
+  deleteDatabaseBackup,
   getAssetInfo,
   getConfig,
   getConfigDefaults,
   getQueuesLegacy,
+  listDatabaseBackups,
   login,
   runQueueCommandLegacy,
   scanLibrary,
@@ -52,11 +57,15 @@ import {
 import { BrowserContext } from '@playwright/test';
 import { exec, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdtemp } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { dirname, resolve } from 'node:path';
+import { dirname, join, resolve } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
 import { promisify } from 'node:util';
+import { createGzip } from 'node:zlib';
 import pg from 'pg';
 import { io, type Socket } from 'socket.io-client';
 import { loginDto, signupDto } from 'src/fixtures';
@@ -84,8 +93,9 @@ export const asBearerAuth = (accessToken: string) => ({ Authorization: `Bearer $
 export const asKeyAuth = (key: string) => ({ 'x-api-key': key });
 export const immichCli = (args: string[]) =>
   executeCommand('pnpm', ['exec', 'immich', '-d', `/${tempDir}/immich/`, ...args], { cwd: '../cli' }).promise;
-export const immichAdmin = (args: string[]) =>
-  executeCommand('docker', ['exec', '-i', 'immich-e2e-server', '/bin/bash', '-c', `immich-admin ${args.join(' ')}`]);
+export const dockerExec = (args: string[]) =>
+  executeCommand('docker', ['exec', '-i', 'immich-e2e-server', '/bin/bash', '-c', args.join(' ')]);
+export const immichAdmin = (args: string[]) => dockerExec([`immich-admin ${args.join(' ')}`]);
 export const specialCharStrings = ["'", '"', ',', '{', '}', '*'];
 export const TEN_TIMES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
 
@@ -149,12 +159,26 @@ const onEvent = ({ event, id }: { event: EventType; id: string }) => {
 };
 
 export const utils = {
+  connectDatabase: async () => {
+    if (!client) {
+      client = new pg.Client(dbUrl);
+      client.on('end', () => (client = null));
+      client.on('error', () => (client = null));
+      await client.connect();
+    }
+
+    return client;
+  },
+
+  disconnectDatabase: async () => {
+    if (client) {
+      await client.end();
+    }
+  },
+
   resetDatabase: async (tables?: string[]) => {
     try {
-      if (!client) {
-        client = new pg.Client(dbUrl);
-        await client.connect();
-      }
+      client = await utils.connectDatabase();
 
       tables = tables || [
         // TODO e2e test for deleting a stack, since it is quite complex
@@ -481,6 +505,9 @@ export const utils = {
   tagAssets: (accessToken: string, tagId: string, assetIds: string[]) =>
     tagAssets({ id: tagId, bulkIdsDto: { ids: assetIds } }, { headers: asBearerAuth(accessToken) }),
 
+  createJob: async (accessToken: string, jobCreateDto: JobCreateDto) =>
+    createJob({ jobCreateDto }, { headers: asBearerAuth(accessToken) }),
+
   queueCommand: async (accessToken: string, name: QueueName, queueCommandDto: QueueCommandDto) =>
     runQueueCommandLegacy({ name, queueCommandDto }, { headers: asBearerAuth(accessToken) }),
 
@@ -559,6 +586,45 @@ export const utils = {
     mkdirSync(`${testAssetDir}/temp`, { recursive: true });
   },
 
+  async move(source: string, dest: string) {
+    return executeCommand('docker', ['exec', 'immich-e2e-server', 'mv', source, dest]).promise;
+  },
+
+  createBackup: async (accessToken: string) => {
+    await utils.createJob(accessToken, {
+      name: ManualJobName.BackupDatabase,
+    });
+
+    return utils.poll(
+      () => request(app).get('/admin/database-backups').set('Authorization', `Bearer ${accessToken}`),
+      ({ status, body }) => status === 200 && body.backups.length === 1,
+      ({ body }) => body.backups[0].filename,
+    );
+  },
+
+  resetBackups: async (accessToken: string) => {
+    const { backups } = await listDatabaseBackups({ headers: asBearerAuth(accessToken) });
+
+    const backupFiles = backups.map((b) => b.filename);
+    await deleteDatabaseBackup(
+      { databaseBackupDeleteDto: { backups: backupFiles } },
+      { headers: asBearerAuth(accessToken) },
+    );
+  },
+
+  prepareTestBackup: async (generate: 'empty' | 'corrupted') => {
+    const dir = await mkdtemp(join(tmpdir(), 'test-'));
+    const fn = join(dir, 'file');
+
+    const sql = Readable.from(generate === 'corrupted' ? 'IM CORRUPTED;' : 'SELECT 1;');
+    const gzip = createGzip();
+    const writeStream = createWriteStream(fn);
+    await pipeline(sql, gzip, writeStream);
+
+    await executeCommand('docker', ['cp', fn, `immich-e2e-server:/data/backups/development-${generate}.sql.gz`])
+      .promise;
+  },
+
   resetAdminConfig: async (accessToken: string) => {
     const defaultConfig = await getConfigDefaults({ headers: asBearerAuth(accessToken) });
     await updateConfig({ systemConfigDto: defaultConfig }, { headers: asBearerAuth(accessToken) });
@@ -601,6 +667,25 @@ export const utils = {
     await utils.waitForQueueFinish(accessToken, 'sidecar');
     await utils.waitForQueueFinish(accessToken, 'metadataExtraction');
   },
+
+  async poll<T>(cb: () => Promise<T>, validate: (value: T) => boolean, map?: (value: T) => any) {
+    let timeout = 0;
+    while (true) {
+      try {
+        const data = await cb();
+        if (validate(data)) {
+          return map ? map(data) : data;
+        }
+        timeout++;
+        if (timeout >= 10) {
+          throw 'Could not clean up test.';
+        }
+        await new Promise((resolve) => setTimeout(resolve, 5e2));
+      } catch {
+        // no-op
+      }
+    }
+  },
 };
 
 utils.initSdk();