feat: restore database backups (#23978)

* feat: ProcessRepository#createSpawnDuplexStream

* test: write tests for ProcessRepository#createSpawnDuplexStream

* feat: StorageRepository#createGzip,createGunzip,createPlainReadStream

* feat: backups util (args, create, restore, progress)

* feat: wait on maintenance operation lock on boot

* chore: use backup util from backup.service.ts
test: update backup.service.ts tests with new util

* feat: list/delete backups (maintenance services)

* chore: open api
fix: missing action in cli.service.ts

* chore: add missing repositories to MaintenanceModule

* refactor: move logSecret into module init

* feat: initialise StorageCore in maintenance mode

* feat: authenticate websocket requests in maintenance mode

* test: add mock for new storage fns

* feat: add MaintenanceEphemeralStateRepository
refactor: cache the secret in memory

* test: update service worker tests

* feat: add external maintenance mode status

* feat: synchronised status, restore db action

* test: backup restore service tests

* refactor: DRY end maintenance

* feat: list and delete backup routes

* feat: start action on boot

* fix: should set status on restore end

* refactor: add maintenanceStore to hold writables

* feat: sync status to web app

* feat: web impl.

* test: various utils for testings

* test: web e2e tests

* test: e2e maintenance spec

* test: update cli spec

* chore: e2e lint

* chore: lint fixes

* chore: lint fixes

* feat: start restore flow route

* test: update e2e tests

* chore: remove neon lights on maintenance action pages

* fix: use 'startRestoreFlow' on onboarding page

* chore: ignore any library folder in `docker/`

* fix: load status on boot

* feat: upload backups

* refactor: permit any .sql(.gz) to be listed/restored

* feat: download backups from list

* fix: permit uploading just .sql files

* feat: restore just .sql files

* fix: don't show backups list if logged out

* feat: system integrity check in restore flow

* test: not providing failed backups in API anymore

* test: util should also not try to use failedBackups

* fix: actually assign inputStream

* test: correct test backup prep.

* fix: ensure task is defined to show error

* test: fix docker cp command

* test: update e2e web spec to select next button

* test: update e2e api tests

* test: refactor timeouts

* chore: remove `showDelete` from maint. settings

* chore: lint

* chore: lint

* fix: make sure backups are correctly sorted for clean up

* test: update service spec

* test: adjust e2e timeout

* test: increase web timeouts for ci

* chore: move gitignore changes

* chore: additional filename validation

* refactor: better typings for integrity API

* feat: higher accuracy progress tracking

* chore: delay lock retry

* refactor: remove old maintenance settings

* refactor: clean up tailwind classes

* refactor: use while loop rather than recursive calls

* test: update service specs

* chore: check canParse too

* chore: lint

* fix: logic error causing infinite loop

* refactor: use <ProgressBar /> from ui library

* fix: create or overwrite file

* chore: i18n pass, update progress bar

* fix: wrong translation string

* chore: update colour variables

* test: update web test for new maint. page

* chore: format, fix key

* test: update tests to be more linter complaint & use new routines

* chore: update onClick -> onAction, title -> breadcrumbs

* fix: use wrench icon in admin settings sidebar

* chore: add translation strings to accordion

* chore: lint

* refactor: move maintenance worker init into service

* refactor: `maintenanceStatus` -> `getMaintenanceStatus`
refactor: `integrityCheck` -> `detectPriorInstall`
chore: add `v2.4.0` version
refactor: `/backups/list` -> `/backups`
refactor: use sendFile in download route
refactor: use separate backups permissions
chore: correct descriptions
refactor: permit handler that doesn't return promise for sendfile

* refactor: move status impl into service
refactor: add active flag to maintenance status

* refactor: split into database backup controller

* test: split api e2e tests and passing

* fix: move end button into authed default maint page

* fix: also show in restore flow

* fix: import getMaintenanceStatus

* test: split web e2e tests

* refactor: ensure detect install is consistently named

* chore: ensure admin for detect install while out of maint.

* refactor: remove state repository

* test: update maint. worker service spec

* test: split backup service spec

* refactor: rename db backup routes

* refactor: instead of param, allow bulk backup deletion

* test: update sdk use in e2e test

* test: correct deleteBackup call

* fix: correct type for serverinstall response dto

* chore: validate filename for deletion

* test: wip

* test: backups no longer take path param

* refactor: scope util to database-backups instead of backups

* fix: update worker controller with new route

* chore: use new admin page actions

* chore: remove stray comment

* test: rename outdated test

* refactor: getter pattern for maintenance secret

* refactor: `createSpawnDuplexStream` -> `spawnDuplexStream`

* refactor: prefer `Object.assign`

* refactor: remove useless try {} block

* refactor: prefer `type Props`
refactor: prefer arrow function

* refactor: use luxon API for minutesAgo

* chore: remove change to gitignore

* refactor: prefer `type Props`

* refactor: remove async from onMount

* refactor: use luxon toRelative for relative time

* refactor: duplicate logic check

* chore: open api

* refactor: begin moving code into web//services

* refactor: don't use template string with $t

* test: use dialog role to match prompt

* refactor: split actions into flow/restore

* test: fix action value

* refactor: move more service calls into web//services

* chore: should void fn return

* chore: bump 2.4.0 to 2.5.0 in controller

* chore: bump 2.4.0 to 2.5.0 in controller

* refactor: use events for web//services

* chore: open api

* chore: open api

* refactor: don't await returned promise

* refactor: remove redundant check

* refactor: add `type: command` to actions

* refactor: split backup entries into own component

* refactor: split restore flow into separate components

* refactor(web): split BackupDelete event

* chore: stylings

* chore: stylings

* fix: don't log query failure on first boot

* feat: support pg_dumpall backups

* feat: display information about each backup

* chore: i18n

* feat: rollback to restore point on migrations failure

* feat: health check after restore

* chore: format

* refactor: split health check into separate function

* refactor: split health into repository
test: write tests covering rollbacks

* fix: omit 'health' requirement from createDbBackup

* test(e2e): rollback test

* fix: wrap text in backup entry

* fix: don't shrink context menu button

* fix: correct CREATE DB syntax for postgres

* test: rename backups generated by test

* feat: add filesize to backup response dto

* feat: restore list

* feat: ui work

* fix: e2e test

* fix: e2e test

* pr feedback

* pr feedback

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
Paul Makles
2026-01-20 15:22:28 +00:00
committed by GitHub
parent ca0d4b283a
commit 61a9d5cbc7
81 changed files with 5585 additions and 391 deletions
@@ -1,35 +0,0 @@
<script lang="ts">
import { handleError } from '$lib/utils/handle-error';
import { MaintenanceAction, setMaintenanceMode } from '@immich/sdk';
import { Button } from '@immich/ui';
import { t } from 'svelte-i18n';
import { fade } from 'svelte/transition';
interface Props {
disabled?: boolean;
}
let { disabled = false }: Props = $props();
async function start() {
try {
await setMaintenanceMode({
setMaintenanceModeDto: {
action: MaintenanceAction.Start,
},
});
} catch (error) {
handleError(error, $t('admin.maintenance_start_error'));
}
}
</script>
<div>
<div in:fade={{ duration: 500 }}>
<div class="ms-4 mt-4 flex items-end gap-4">
<Button shape="round" type="submit" {disabled} size="small" onclick={start}
>{$t('admin.maintenance_start')}</Button
>
</div>
</div>
</div>
@@ -6,7 +6,7 @@
import { sidebarStore } from '$lib/stores/sidebar.svelte';
import type { HeaderButtonActionItem } from '$lib/types';
import { AppShell, AppShellHeader, AppShellSidebar, MenuItemType, NavbarItem, type BreadcrumbItem } from '@immich/ui';
import { mdiAccountMultipleOutline, mdiBookshelf, mdiCog, mdiServer, mdiTrayFull } from '@mdi/js';
import { mdiAccountMultipleOutline, mdiBookshelf, mdiCog, mdiServer, mdiTrayFull, mdiWrench } from '@mdi/js';
import type { Snippet } from 'svelte';
import { t } from 'svelte-i18n';
@@ -32,6 +32,7 @@
<NavbarItem title={$t('external_libraries')} href={Route.libraries()} icon={mdiBookshelf} />
<NavbarItem title={$t('admin.queues')} href={Route.queues()} icon={mdiTrayFull} />
<NavbarItem title={$t('settings')} href={Route.systemSettings()} icon={mdiCog} />
<NavbarItem title={$t('admin.maintenance_settings')} href={Route.systemMaintenance()} icon={mdiWrench} />
<NavbarItem title={$t('server_stats')} href={Route.systemStatistics()} icon={mdiServer} />
</div>
@@ -5,24 +5,27 @@
title?: string;
children?: Snippet;
withHeader?: boolean;
withBackdrop?: boolean;
}
let { title, children, withHeader = true }: Props = $props();
let { title, children, withHeader = true, withBackdrop = true }: Props = $props();
</script>
<section class="min-w-dvw flex min-h-dvh items-center justify-center relative">
<div class="absolute -z-10 w-full h-full flex place-items-center place-content-center">
<img
src={immichLogo}
class="max-w-(--breakpoint-md) mx-auto h-full mb-2 antialiased overflow-hidden"
alt="Immich logo"
/>
<div
class="w-full h-[99%] absolute start-0 top-0 backdrop-blur-[200px] bg-transparent dark:bg-immich-dark-bg/20"
></div>
</div>
{#if withBackdrop}
<div class="absolute -z-10 w-full h-full flex place-items-center place-content-center">
<img
src={immichLogo}
class="max-w-(--breakpoint-md) mx-auto h-full mb-2 antialiased overflow-hidden"
alt="Immich logo"
/>
<div
class="w-full h-[99%] absolute start-0 top-0 backdrop-blur-[200px] bg-transparent dark:bg-immich-dark-bg/20"
></div>
</div>
{/if}
<Card color="secondary" class="w-full max-w-lg border m-2">
<Card color="secondary" class="w-full max-w-xl border m-2">
{#if withHeader}
<CardHeader class="mt-6">
<VStack>
@@ -0,0 +1,125 @@
<script lang="ts">
import OnEvents from '$lib/components/OnEvents.svelte';
import { BackupFileStatus } from '$lib/constants';
import { getDatabaseBackupActions, handleRestoreDatabaseBackup } from '$lib/services/database-backups.service';
import { locale } from '$lib/stores/preferences.store';
import { getBytesWithUnit } from '$lib/utils/byte-units';
import { Button, Card, CardBody, ContextMenuButton, HStack, Icon, Stack, Text } from '@immich/ui';
import { mdiAlertCircle, mdiCheckCircle, mdiDatabaseRefreshOutline } from '@mdi/js';
import { DateTime } from 'luxon';
import { t } from 'svelte-i18n';
type Props = {
filename: string;
filesize: number;
expectedVersion: string;
};
const { filename, filesize, expectedVersion }: Props = $props();
const filesizeText = $derived(getBytesWithUnit(filesize, 1));
const backupDateTime = $derived.by(() => {
const dateMatch = filename.match(/\d+T\d+/);
if (dateMatch) {
return DateTime.fromFormat(dateMatch[0], "yyyyMMdd'T'HHmmss", { zone: 'utc' }).toLocal();
}
return null;
});
const timeDisplay = $derived(backupDateTime?.toLocaleString(DateTime.TIME_SIMPLE));
const relativeTime = $derived(backupDateTime?.toRelative({ locale: $locale }));
const version = $derived(filename.match(/-v(.*)-/)?.[1]);
const status = $derived.by(() => {
if (!version) {
return BackupFileStatus.UnknownVersion;
}
if (version !== expectedVersion) {
return BackupFileStatus.DifferentVersion;
}
return BackupFileStatus.OK;
});
const { Download, Delete } = $derived(getDatabaseBackupActions($t, filename));
let isDeleting = $state(false);
function onBackupDeleteStatus(event: { filename: string; isDeleting: boolean }) {
if (event.filename === filename) {
isDeleting = event.isDeleting;
}
}
</script>
<OnEvents {onBackupDeleteStatus} />
<Card class="dark:bg-dark-900">
<CardBody class="pt-3 pb-4 px-6">
<Stack gap={3} class="grow min-w-0">
<div class="flex justify-between items-center gap-3">
<HStack gap={2} class="min-w-0">
{#if status === BackupFileStatus.OK}
<Icon icon={mdiCheckCircle} size="18" class="text-success" />
{:else if status === BackupFileStatus.DifferentVersion}
<Icon icon={mdiAlertCircle} size="18" class="text-warning" />
{:else}
<Icon icon={mdiAlertCircle} size="18" class="text-danger" />
{/if}
{#if timeDisplay}
<Text class="font-medium" size="small">{timeDisplay}</Text>
{:else}
<Text class="font-medium" size="small">{$t('unknown_date')}</Text>
{/if}
{#if relativeTime}
<div class="flex items-center gap-2">
<div class="w-1 h-1 bg-light-500"></div>
<Text size="tiny" color="muted">{relativeTime}</Text>
</div>
{/if}
</HStack>
<HStack gap={1}>
<Button size="small" onclick={() => handleRestoreDatabaseBackup(filename)} disabled={isDeleting}
>{$t('restore')}</Button
>
<ContextMenuButton
disabled={isDeleting}
position="top-right"
aria-label={$t('open')}
items={[Download, Delete]}
/>
</HStack>
</div>
<HStack>
<Icon icon={mdiDatabaseRefreshOutline} size="16" color="gray" />
<Text size="small" class="break-all font-mono">{filename}</Text>
</HStack>
{#if status === BackupFileStatus.UnknownVersion}
<Text size="small" color="danger">
{$t('admin.maintenance_restore_backup_unknown_version')}
</Text>
{:else if status === BackupFileStatus.DifferentVersion}
<Text size="small" color="warning">
{$t('admin.maintenance_restore_backup_different_version')}
</Text>
{/if}
<HStack gap={8}>
<div class="flex gap-1">
<Text size="tiny" color="muted">{$t('version')}:</Text>
<Text size="tiny" fontWeight="medium">{version ? `v${version}` : $t('unknown')}</Text>
</div>
<div class="flex gap-1">
<Text size="tiny" color="muted">{$t('size')}:</Text>
<Text size="tiny" fontWeight="medium">{filesizeText[0]} {filesizeText[1]}</Text>
</div>
</HStack>
</Stack>
</CardBody>
</Card>
@@ -0,0 +1,135 @@
<script lang="ts">
import HeaderActionButton from '$lib/components/HeaderActionButton.svelte';
import MaintenanceBackupEntry from '$lib/components/maintenance/MaintenanceBackupEntry.svelte';
import OnEvents from '$lib/components/OnEvents.svelte';
import { handleUploadDatabaseBackup } from '$lib/services/database-backups.service';
import type { DatabaseBackupDto } from '@immich/sdk';
import { listDatabaseBackups } from '@immich/sdk';
import { Card, CardBody, HStack, Icon, ProgressBar, Stack, Text } from '@immich/ui';
import { mdiCalendar, mdiTrayArrowUp } from '@mdi/js';
import { DateTime } from 'luxon';
import { onMount } from 'svelte';
import { t } from 'svelte-i18n';
type Props = {
backups?: DatabaseBackupDto[];
expectedVersion: string;
};
let props: Props = $props();
let backups = $state(props.backups ?? []);
async function reloadBackups() {
const result = await listDatabaseBackups();
backups = result.backups;
}
onMount(() => {
if (!props.backups) {
void reloadBackups();
}
});
let uploadProgress = $state(-1);
function onBackupDeleted(event: { filename: string }) {
backups = backups.filter((backup) => backup.filename !== event.filename);
}
function onBackupUpload(event: { progress: number; isComplete: boolean }) {
uploadProgress = event.progress;
if (event.isComplete) {
void reloadBackups();
}
}
const groupedBackups = $derived.by(() => {
// eslint-disable-next-line svelte/prefer-svelte-reactivity
const groups = new Map<string, { date: DateTime; backups: DatabaseBackupDto[] }>();
const unknownDateKey = $t('unknown_date');
for (const backup of backups) {
const dateMatch = backup.filename.match(/\d+T\d+/);
let dateKey: string;
let dt: DateTime;
if (dateMatch) {
dt = DateTime.fromFormat(dateMatch[0], "yyyyMMdd'T'HHmmss", { zone: 'utc' });
dateKey = dt.toFormat('LLLL d, yyyy');
} else {
dt = DateTime.fromMillis(0);
dateKey = unknownDateKey;
}
if (!groups.has(dateKey)) {
groups.set(dateKey, { date: dt.startOf('day'), backups: [] });
}
groups.get(dateKey)!.backups.push(backup);
}
// Sort by date descending (newest first), but put unknown date at the top
const sortedEntries = [...groups.entries()].sort((a, b) => {
if (a[0] === unknownDateKey) {
return -1;
}
if (b[0] === unknownDateKey) {
return 1;
}
return b[1].date.toMillis() - a[1].date.toMillis();
});
return new Map(sortedEntries.map(([key, value]) => [key, value.backups]));
});
</script>
<OnEvents {onBackupDeleted} {onBackupUpload} />
<Stack gap={4} class="mt-4 text-left">
<Card color="info">
<CardBody>
{#if uploadProgress === -1}
<div class="flex justify-between items-center">
<div class="flex gap-2 items-end w-max">
<Icon icon={mdiTrayArrowUp} size="20" class="text-muted"></Icon>
<Text class="grow">{$t('admin.maintenance_upload_backup')}</Text>
</div>
<HeaderActionButton
action={{
color: 'primary',
title: $t('select_from_computer'),
onAction: handleUploadDatabaseBackup,
}}
/>
</div>
{:else}
<HStack gap={8}>
<Text class="grow">{$t('asset_uploading')}</Text>
<ProgressBar progress={uploadProgress} size="tiny" />
</HStack>
{/if}
</CardBody>
</Card>
<hr />
{#each [...groupedBackups.entries()] as [dateGroup, groupBackups] (dateGroup)}
<Stack gap={2}>
<div class="mt-5 mb-1">
<div class="bg-primary-50 flex gap-2 px-4 py-2 rounded-xl w-max place-items-center">
<Icon icon={mdiCalendar} size="18" />
<Text size="small" fontWeight="medium" color="muted">{dateGroup}</Text>
</div>
</div>
{#each groupBackups as backup (backup.filename)}
<MaintenanceBackupEntry
filename={backup.filename}
filesize={backup.filesize}
expectedVersion={props.expectedVersion}
/>
{/each}
</Stack>
{/each}
</Stack>
@@ -0,0 +1,19 @@
<script lang="ts">
import RestoreFlowDetectInstall from '$lib/components/maintenance/restore-flow/RestoreFlowDetectInstall.svelte';
import RestoreFlowSelectBackup from '$lib/components/maintenance/restore-flow/RestoreFlowSelectBackup.svelte';
type Props = {
end: () => void;
expectedVersion: string;
};
const { end, expectedVersion }: Props = $props();
let stage = $state(0);
</script>
{#if stage === 0}
<RestoreFlowDetectInstall next={() => stage++} {end} />
{:else}
<RestoreFlowSelectBackup previous={() => stage--} {end} {expectedVersion} />
{/if}
@@ -0,0 +1,98 @@
<script lang="ts">
import { detectPriorInstall, type MaintenanceDetectInstallResponseDto } from '@immich/sdk';
import { Button, Heading, HStack, Icon, Stack, Text } from '@immich/ui';
import { mdiAlert, mdiArrowRight, mdiCheck, mdiClose, mdiRefresh } from '@mdi/js';
import { onMount } from 'svelte';
import { t } from 'svelte-i18n';
type Props = {
next: () => void;
end: () => void;
};
const { next, end }: Props = $props();
let detectedInstall: MaintenanceDetectInstallResponseDto | undefined = $state();
const reload = async () => {
detectedInstall = await detectPriorInstall();
};
const getLibraryFolderCheckStatus = (writable: boolean, readable: boolean) => {
if (writable) {
return $t('maintenance_restore_library_folder_pass');
} else if (readable) {
return $t('maintenance_restore_library_folder_write_fail');
} else {
return $t('maintenance_restore_library_folder_read_fail');
}
};
onMount(() => reload());
</script>
<Heading size="large" color="primary" tag="h1">{$t('maintenance_restore_library')}</Heading>
<Text>{$t('maintenance_restore_library_description')}</Text>
<div class="bg-white dark:bg-black w-full m-4 p-6 rounded-xl border border-light-300">
<Stack>
{#if detectedInstall}
{#each detectedInstall.storage as { folder, readable, writable } (folder)}
<HStack>
<Icon icon={writable ? mdiCheck : mdiClose} class={writable ? 'text-success' : 'text-danger'} />
<Text>{folder} ({getLibraryFolderCheckStatus(writable, readable)})</Text>
</HStack>
{/each}
{#each detectedInstall.storage as { folder, files } (folder)}
{#if folder !== 'backups'}
<HStack class="items-start">
<Icon
class={`mt-1 ${files ? 'text-success' : folder === 'profile' || folder === 'upload' ? 'text-danger' : 'text-warning'}`}
icon={files ? mdiCheck : folder === 'profile' || folder === 'upload' ? mdiClose : mdiAlert}
/>
<Stack gap={0} class="items-start">
<Text>
{#if files}
{$t('maintenance_restore_library_folder_has_files', {
values: {
folder,
count: files,
},
})}
{:else}
{$t('maintenance_restore_library_folder_no_files', {
values: {
folder,
},
})}
{/if}
</Text>
{#if !files}
{#if folder === 'profile' || folder === 'upload'}
<Text variant="italic">{$t('maintenance_restore_library_hint_missing_files')}</Text>
{/if}
{#if folder === 'encoded-video' || folder === 'thumbs'}
<Text variant="italic">{$t('maintenance_restore_library_hint_regenerate_later')}</Text>
{/if}
{#if folder === 'library'}
<Text variant="italic">{$t('maintenance_restore_library_hint_storage_template_missing_files')}</Text>
{/if}
{/if}
</Stack>
</HStack>
{/if}
{/each}
<Button leadingIcon={mdiRefresh} variant="ghost" onclick={reload}>{$t('refresh')}</Button>
{:else}
<HStack>
<Icon icon={mdiRefresh} color="rgb(var(--immich-ui-primary))" />
<Text>{$t('maintenance_restore_library_loading')}</Text>
</HStack>
{/if}
</Stack>
</div>
<Text>{$t('maintenance_restore_library_confirm')}</Text>
<HStack>
<Button onclick={end} variant="ghost">{$t('cancel')}</Button>
<Button onclick={next} trailingIcon={mdiArrowRight}>{$t('next')}</Button>
</HStack>
@@ -0,0 +1,23 @@
<script lang="ts">
import { Button, Heading, HStack, Scrollable } from '@immich/ui';
import { mdiArrowLeft } from '@mdi/js';
import { t } from 'svelte-i18n';
import MaintenanceBackupsList from '../MaintenanceBackupsList.svelte';
type Props = {
previous: () => void;
end: () => void;
expectedVersion: string;
};
const { previous, end, expectedVersion }: Props = $props();
</script>
<Heading size="large" color="primary" tag="h1">{$t('maintenance_restore_from_backup')}</Heading>
<Scrollable class="max-h-120 bg-white dark:bg-black p-4 rounded-2xl border border-light-300 w-full">
<MaintenanceBackupsList {expectedVersion} />
</Scrollable>
<HStack>
<Button onclick={end} variant="ghost">{$t('cancel')}</Button>
<Button onclick={previous} variant="ghost" leadingIcon={mdiArrowLeft}>{$t('back')}</Button>
</HStack>