hls service and controller

open-api/immich-openapi-specs.json

@@ -4300,6 +4300,351 @@
         "x-immich-state": "Stable"
       }
     },
+    "/assets/{id}/video/stream/master.m3u8": {
+      "get": {
+        "description": "Returns an HLS master playlist with all available variants for the asset.",
+        "operationId": "getMasterPlaylist",
+        "parameters": [
+          {
+            "name": "id",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "key",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/vnd.apple.mpegurl": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "Get HLS master playlist",
+        "tags": [
+          "Assets"
+        ],
+        "x-immich-history": [
+          {
+            "version": "v3",
+            "state": "Added"
+          },
+          {
+            "version": "v3",
+            "state": "Alpha"
+          }
+        ],
+        "x-immich-permission": "asset.view",
+        "x-immich-state": "Alpha"
+      }
+    },
+    "/assets/{id}/video/stream/{sessionId}": {
+      "delete": {
+        "description": "Releases server resources for the streaming session.",
+        "operationId": "endSession",
+        "parameters": [
+          {
+            "name": "id",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "key",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "sessionId",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "End HLS streaming session",
+        "tags": [
+          "Assets"
+        ],
+        "x-immich-history": [
+          {
+            "version": "v3",
+            "state": "Added"
+          },
+          {
+            "version": "v3",
+            "state": "Alpha"
+          }
+        ],
+        "x-immich-permission": "asset.view",
+        "x-immich-state": "Alpha"
+      }
+    },
+    "/assets/{id}/video/stream/{sessionId}/{variantIndex}/playlist.m3u8": {
+      "get": {
+        "description": "Returns an HLS media playlist for one variant of the streaming session.",
+        "operationId": "getMediaPlaylist",
+        "parameters": [
+          {
+            "name": "id",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "key",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "sessionId",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "variantIndex",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "minimum": 0,
+              "maximum": 9007199254740991,
+              "type": "integer"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/vnd.apple.mpegurl": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "Get HLS media playlist",
+        "tags": [
+          "Assets"
+        ],
+        "x-immich-history": [
+          {
+            "version": "v3",
+            "state": "Added"
+          },
+          {
+            "version": "v3",
+            "state": "Alpha"
+          }
+        ],
+        "x-immich-permission": "asset.view",
+        "x-immich-state": "Alpha"
+      }
+    },
+    "/assets/{id}/video/stream/{sessionId}/{variantIndex}/{filename}": {
+      "get": {
+        "description": "Streams an HLS init segment (init.mp4) or media segment (seg_N.m4s).",
+        "operationId": "getSegment",
+        "parameters": [
+          {
+            "name": "filename",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "pattern": "^(init\\.mp4|seg_\\d+\\.m4s)$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "id",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "key",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "sessionId",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "format": "uuid",
+              "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$",
+              "type": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "variantIndex",
+            "required": true,
+            "in": "path",
+            "schema": {
+              "minimum": 0,
+              "maximum": 9007199254740991,
+              "type": "integer"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "content": {
+              "application/octet-stream": {
+                "schema": {
+                  "format": "binary",
+                  "type": "string"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "summary": "Get HLS segment or init file",
+        "tags": [
+          "Assets"
+        ],
+        "x-immich-history": [
+          {
+            "version": "v3",
+            "state": "Added"
+          },
+          {
+            "version": "v3",
+            "state": "Alpha"
+          }
+        ],
+        "x-immich-permission": "asset.view",
+        "x-immich-state": "Alpha"
+      }
+    },
     "/auth/admin-sign-up": {
       "post": {
         "description": "Create the first admin user in the system.",
@@ -17897,6 +18242,7 @@
           "LibrarySyncFilesQueueAll",
           "LibrarySyncFiles",
           "LibraryScanQueueAll",
+          "HlsSessionCleanup",
           "MemoryCleanup",
           "MemoryGenerate",
           "NotificationsCleanup",
@@ -24095,6 +24441,9 @@
             "description": "Preset",
             "type": "string"
           },
+          "realtime": {
+            "$ref": "#/components/schemas/SystemConfigFFmpegRealtimeDto"
+          },
           "refs": {
             "description": "References",
             "maximum": 6,
@@ -24145,6 +24494,7 @@
           "maxBitrate",
           "preferredHwDevice",
           "preset",
+          "realtime",
           "refs",
           "targetAudioCodec",
           "targetResolution",
@@ -24157,6 +24507,18 @@
         ],
         "type": "object"
       },
+      "SystemConfigFFmpegRealtimeDto": {
+        "properties": {
+          "enabled": {
+            "description": "Enable real-time HLS transcoding (alpha)",
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "enabled"
+        ],
+        "type": "object"
+      },
       "SystemConfigFacesDto": {
         "properties": {
           "import": {

server/src/config.ts

@@ -45,6 +45,9 @@ export type SystemConfig = {
     accel: TranscodeHardwareAcceleration;
     accelDecode: boolean;
     tonemap: ToneMapping;
+    realtime: {
+      enabled: boolean;
+    };
   };
   job: Record<ConcurrentQueueName, { concurrency: number }>;
   logging: {
@@ -224,6 +227,9 @@ export const defaults = Object.freeze<SystemConfig>({
     tonemap: ToneMapping.Hable,
     accel: TranscodeHardwareAcceleration.Disabled,
     accelDecode: false,
+    realtime: {
+      enabled: false,
+    },
   },
   job: {
     [QueueName.BackgroundTask]: { concurrency: 5 },

server/src/constants.ts

@@ -1,7 +1,15 @@
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { SemVer } from 'semver';
-import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
+import {
+  ApiTag,
+  AudioCodec,
+  DatabaseExtension,
+  ExifOrientation,
+  TranscodeHardwareAcceleration,
+  VectorIndex,
+  VideoCodec,
+} from 'src/enum';
 
 export const IMMICH_SERVER_START = 'Immich Server is listening';
 
@@ -203,3 +211,35 @@ export const AUDIO_ENCODER: Record<AudioCodec, string> = {
   [AudioCodec.Opus]: 'libopus',
   [AudioCodec.PcmS16le]: 'pcm_s16le',
 };
+
+export const SUPPORTED_HWA_CODECS: Record<TranscodeHardwareAcceleration, VideoCodec[]> = {
+  [TranscodeHardwareAcceleration.Nvenc]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Av1],
+  [TranscodeHardwareAcceleration.Qsv]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
+  [TranscodeHardwareAcceleration.Vaapi]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
+  [TranscodeHardwareAcceleration.Rkmpp]: [VideoCodec.H264, VideoCodec.Hevc],
+  [TranscodeHardwareAcceleration.Disabled]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
+};
+
+export const HLS_BACKPRESSURE_PAUSE_SEGMENTS = 30;
+export const HLS_BACKPRESSURE_RESUME_SEGMENTS = 15;
+export const HLS_INACTIVITY_SWEEP_INTERVAL_MS = 60 * 1000;
+export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;
+export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000;
+export const HLS_SEGMENT_DURATION = 2;
+export const HLS_VARIANTS = [
+  { resolution: 480, codec: VideoCodec.Av1, bitrate: 1_000_000, codecString: 'av01.0.04M.08' },
+  { resolution: 480, codec: VideoCodec.Hevc, bitrate: 1_200_000, codecString: 'hvc1.1.6.L90.B0' },
+  { resolution: 480, codec: VideoCodec.H264, bitrate: 2_500_000, codecString: 'avc1.64001e' },
+  { resolution: 720, codec: VideoCodec.Av1, bitrate: 2_000_000, codecString: 'av01.0.08M.08' },
+  { resolution: 720, codec: VideoCodec.Hevc, bitrate: 2_500_000, codecString: 'hvc1.1.6.L93.B0' },
+  { resolution: 720, codec: VideoCodec.H264, bitrate: 5_000_000, codecString: 'avc1.64001f' },
+  { resolution: 1080, codec: VideoCodec.Av1, bitrate: 4_000_000, codecString: 'av01.0.09M.08' },
+  { resolution: 1080, codec: VideoCodec.Hevc, bitrate: 4_500_000, codecString: 'hvc1.1.6.L120.B0' },
+  { resolution: 1080, codec: VideoCodec.H264, bitrate: 8_000_000, codecString: 'avc1.640028' },
+  { resolution: 1440, codec: VideoCodec.Av1, bitrate: 7_000_000, codecString: 'av01.0.12M.08' },
+  { resolution: 1440, codec: VideoCodec.Hevc, bitrate: 8_000_000, codecString: 'hvc1.2.4.L150.B0' },
+  { resolution: 1440, codec: VideoCodec.H264, bitrate: 16_000_000, codecString: 'avc1.640032' },
+];
+export const HLS_VERSION = 6;
+export const PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl';
+export const SEGMENT_FILENAME_REGEX = /^seg_(\d+)\.m4s$/;

server/src/controllers/index.ts

@@ -35,6 +35,7 @@ import { TimelineController } from 'src/controllers/timeline.controller';
 import { TrashController } from 'src/controllers/trash.controller';
 import { UserAdminController } from 'src/controllers/user-admin.controller';
 import { UserController } from 'src/controllers/user.controller';
+import { VideoStreamController } from 'src/controllers/video-stream.controller';
 import { ViewController } from 'src/controllers/view.controller';
 import { WorkflowController } from 'src/controllers/workflow.controller';
 
@@ -76,6 +77,7 @@ export const controllers = [
   TrashController,
   UserAdminController,
   UserController,
+  VideoStreamController,
   ViewController,
   WorkflowController,
 ];

server/src/controllers/video-stream.controller.ts

@@ -0,0 +1,79 @@
+import { Controller, Delete, Get, Header, HttpCode, HttpStatus, Next, Param, Res } from '@nestjs/common';
+import { ApiProduces, ApiTags } from '@nestjs/swagger';
+import { NextFunction, Response } from 'express';
+import { PLAYLIST_CONTENT_TYPE } from 'src/constants';
+import { Endpoint, HistoryBuilder } from 'src/decorators';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { HlsSegmentParamDto, HlsSessionParamDto, HlsVariantParamDto } from 'src/dtos/streaming.dto';
+import { ApiTag, Permission, RouteKey } from 'src/enum';
+import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard';
+import { LoggingRepository } from 'src/repositories/logging.repository';
+import { HlsService } from 'src/services/hls.service';
+import { sendFile } from 'src/utils/file';
+import { UUIDParamDto } from 'src/validation';
+
+@ApiTags(ApiTag.Assets)
+@Controller(RouteKey.Asset)
+export class VideoStreamController {
+  constructor(
+    private logger: LoggingRepository,
+    private service: HlsService,
+  ) {}
+
+  @Get(':id/video/stream/master.m3u8')
+  @Authenticated({ permission: Permission.AssetView, sharedLink: true })
+  @Header('Cache-Control', 'no-cache')
+  @Header('Content-Type', PLAYLIST_CONTENT_TYPE)
+  @ApiProduces(PLAYLIST_CONTENT_TYPE)
+  @Endpoint({
+    summary: 'Get HLS master playlist',
+    description: 'Returns an HLS master playlist with all available variants for the asset.',
+    history: new HistoryBuilder().added('v3').alpha('v3'),
+  })
+  getMasterPlaylist(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto) {
+    return this.service.getMasterPlaylist(auth, id);
+  }
+
+  @Get(':id/video/stream/:sessionId/:variantIndex/playlist.m3u8')
+  @Authenticated({ permission: Permission.AssetView, sharedLink: true })
+  @Header('Cache-Control', 'no-cache')
+  @Header('Content-Type', PLAYLIST_CONTENT_TYPE)
+  @ApiProduces(PLAYLIST_CONTENT_TYPE)
+  @Endpoint({
+    summary: 'Get HLS media playlist',
+    description: 'Returns an HLS media playlist for one variant of the streaming session.',
+    history: new HistoryBuilder().added('v3').alpha('v3'),
+  })
+  getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) {
+    return this.service.getMediaPlaylist(auth, id, sessionId);
+  }
+
+  @Get(':id/video/stream/:sessionId/:variantIndex/:filename')
+  @FileResponse()
+  @Authenticated({ permission: Permission.AssetView, sharedLink: true })
+  @Endpoint({
+    summary: 'Get HLS segment or init file',
+    description: 'Streams an HLS init segment (init.mp4) or media segment (seg_N.m4s).',
+    history: new HistoryBuilder().added('v3').alpha('v3'),
+  })
+  async getSegment(
+    @Auth() auth: AuthDto,
+    @Param() { id, sessionId, variantIndex, filename }: HlsSegmentParamDto,
+    @Res() res: Response,
+    @Next() next: NextFunction,
+  ) {
+    await sendFile(res, next, () => this.service.getSegment(auth, id, sessionId, variantIndex, filename), this.logger);
+  }
+
+  @Delete(':id/video/stream/:sessionId')
+  @HttpCode(HttpStatus.NO_CONTENT)
+  @Authenticated({ permission: Permission.AssetView, sharedLink: true })
+  @Endpoint({
+    summary: 'End HLS streaming session',
+    description: 'Releases server resources for the streaming session.',
+    history: new HistoryBuilder().added('v3').alpha('v3'),
+  })
+  async endSession(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsSessionParamDto) {
+    await this.service.endSession(auth, id, sessionId);
+  }
+}

server/src/dtos/streaming.dto.ts

@@ -0,0 +1,26 @@
+import { createZodDto } from 'nestjs-zod';
+import z from 'zod';
+
+const HlsSessionParamSchema = z.object({
+  id: z.uuidv4(),
+  sessionId: z.uuidv4(),
+});
+
+export class HlsSessionParamDto extends createZodDto(HlsSessionParamSchema) {}
+
+const HlsVariantParamSchema = z.object({
+  id: z.uuidv4(),
+  sessionId: z.uuidv4(),
+  variantIndex: z.coerce.number().int().min(0),
+});
+
+export class HlsVariantParamDto extends createZodDto(HlsVariantParamSchema) {}
+
+const HlsSegmentParamSchema = z.object({
+  id: z.uuidv4(),
+  sessionId: z.uuidv4(),
+  variantIndex: z.coerce.number().int().min(0),
+  filename: z.string().regex(/^(init\.mp4|seg_\d+\.m4s)$/, { error: 'Invalid HLS segment filename' }),
+});
+
+export class HlsSegmentParamDto extends createZodDto(HlsSegmentParamSchema) {}

server/src/dtos/system-config.dto.ts

@@ -83,6 +83,11 @@ const SystemConfigFFmpegSchema = z
     accel: TranscodeHardwareAccelerationSchema,
     accelDecode: configBool.describe('Accelerated decode'),
     tonemap: ToneMappingSchema,
+    realtime: z
+      .object({
+        enabled: configBool.describe('Enable real-time HLS transcoding (alpha)'),
+      })
+      .meta({ id: 'SystemConfigFFmpegRealtimeDto' }),
   })
   .meta({ id: 'SystemConfigFFmpegDto' });
 

server/src/enum.ts

@@ -445,11 +445,7 @@ export enum VideoCodec {
 
 export const VideoCodecSchema = z.enum(VideoCodec).describe('Target video codec').meta({ id: 'VideoCodec' });
 
-export enum VideoSegmentCodec {
+export type VideoSegmentCodec = VideoCodec.Av1 | VideoCodec.Hevc | VideoCodec.H264;
-  Av1 = 'av1',
-  Hevc = 'hevc',
-  H264 = 'h264',
-}
 
 export enum AudioCodec {
   Mp3 = 'mp3',

server/src/repositories/event.repository.ts

@@ -92,6 +92,14 @@ type EventMap = {
 
   AuthChangePassword: [{ userId: string; currentSessionId?: string; invalidateSessions?: boolean }];
 
+  // hls streaming events
+  HlsSegmentRequest: [{ sessionId: string; assetId: string; variantIndex: number; segmentIndex: number }];
+  HlsSegmentResult: [{ sessionId: string; variantIndex: number; segmentIndex: number; error?: string }];
+  HlsHeartbeat: [{ sessionId: string; variantIndex?: number; segmentIndex?: number }];
+  HlsSessionRequest: [{ sessionId: string; assetId: string; ownerId: string }];
+  HlsSessionResult: [{ sessionId: string; error?: string }];
+  HlsSessionEnd: [{ sessionId: string }];
+
   // websocket events
   WebsocketConnect: [{ userId: string }];
 };

server/src/repositories/media.repository.ts

@@ -509,18 +509,43 @@ export class MediaRepository {
     return this.parseInt(b.bit_rate) - this.parseInt(a.bit_rate);
   }
 
+  /* Ported from https://code.ffmpeg.org/FFmpeg/FFmpeg/src/branch/master/fftools/ffmpeg_filter.c
+   * SPDX-License-Identifier: LGPL-2.1-or-later
+   * Copyright (c) FFmpeg authors and contributors — https://ffmpeg.org/
+   * Modifications: TS port operating on probe-derived packet metadata rather than decoded AVFrames. */
   private cfrOutputFrames(packets: { pts: number; duration: number }[], slotsPerTick: number) {
-    // Packets may be out of PTS order due to B-frames
     packets.sort((a, b) => a.pts - b.pts);
     const firstPts = packets[0].pts;
     let outputFrames = 0;
     let nextPts = 0;
+    const history = [0, 0, 0];
     for (const pkt of packets) {
-      const delta = (pkt.pts - firstPts) * slotsPerTick - nextPts + pkt.duration * slotsPerTick;
+      const syncIpts = (pkt.pts - firstPts) * slotsPerTick;
-      const nb = delta < -1.1 ? 0 : delta > 1.1 ? Math.round(delta) : 1;
+      const duration = pkt.duration * slotsPerTick;
+      let delta0 = syncIpts - nextPts;
+      const delta = delta0 + duration;
+
+      if (delta0 < 0 && delta > 0) {
+        delta0 = 0;
+      }
+
+      let nb = 1;
+      let nbPrev = 0;
+      if (delta < -1.1) {
+        nb = 0;
+      } else if (delta > 1.1) {
+        nb = Math.round(delta);
+        if (delta0 > 1.1) {
+          nbPrev = Math.round(delta0 - 0.6);
+        }
+      }
       outputFrames += nb;
       nextPts += nb;
+      history[2] = history[1];
+      history[1] = history[0];
+      history[0] = nbPrev;
     }
-    return outputFrames;
+    const median = history.sort((a, b) => a - b)[1];
+    return outputFrames + median;
   }
 }

server/src/repositories/video-stream.repository.ts

@@ -8,6 +8,7 @@ import {
   VideoStreamSessionTable,
   VideoStreamVariantTable,
 } from 'src/schema/tables/video-stream.table';
+import { withAudioStream, withVideoFormat, withVideoPackets, withVideoStream } from 'src/utils/database';
 
 @Injectable()
 export class VideoStreamRepository {
@@ -27,7 +28,12 @@ export class VideoStreamRepository {
 
   @GenerateSql({ params: [DummyValue.UUID] })
   getSession(id: string) {
-    return this.db.selectFrom('video_stream_session').selectAll().where('id', '=', id).executeTakeFirst();
+    return this.db
+      .selectFrom('video_stream_session')
+      .selectAll()
+      .where('id', '=', id)
+      .where('expiresAt', '>', new Date())
+      .executeTakeFirst();
   }
 
   @GenerateSql({ params: [DummyValue.UUID] })
@@ -47,7 +53,12 @@ export class VideoStreamRepository {
 
   @GenerateSql()
   getExpiredSessions() {
-    return this.db.selectFrom('video_stream_session').select(['id']).where('expiresAt', '<=', new Date()).execute();
+    return this.db
+      .selectFrom('video_stream_session')
+      .innerJoin('asset', 'asset.id', 'video_stream_session.assetId')
+      .select(['video_stream_session.id', 'asset.ownerId'])
+      .where('video_stream_session.expiresAt', '<=', new Date())
+      .execute();
   }
 
   @GenerateSql({ params: [DummyValue.UUID, DummyValue.DATE] })
@@ -59,4 +70,50 @@ export class VideoStreamRepository {
   async deleteSession(id: string) {
     await this.db.deleteFrom('video_stream_session').where('id', '=', id).execute();
   }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  async getForMasterPlaylist(id: string) {
+    return this.db
+      .selectFrom('asset')
+      .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+      .where('asset.id', '=', id)
+      .innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
+      .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
+      .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+      .select((eb) => withVideoPackets(eb).$notNull().as('packets'))
+      .executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
+  async getForMediaPlaylist(id: string, sessionId: string) {
+    return this.db
+      .selectFrom('asset')
+      .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+      .innerJoin('video_stream_session', 'asset.id', 'video_stream_session.assetId')
+      .where('asset.id', '=', id)
+      .where('video_stream_session.id', '=', sessionId)
+      .where('video_stream_session.expiresAt', '>', new Date())
+      .innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
+      .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
+      .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+      .select((eb) => withVideoPackets(eb).$notNull().as('packets'))
+      .executeTakeFirst();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  async getForTranscoding(id: string) {
+    return this.db
+      .selectFrom('asset')
+      .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+      .where('asset.id', '=', id)
+      .leftJoin('asset_audio', 'asset.id', 'asset_audio.assetId')
+      .innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
+      .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
+      .select('asset.originalPath')
+      .select((eb) => withAudioStream(eb).as('audioStream'))
+      .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+      .select((eb) => withVideoFormat(eb).$notNull().as('format'))
+      .select((eb) => withVideoPackets(eb).$notNull().as('packets'))
+      .executeTakeFirst();
+  }
 }

server/src/repositories/websocket.repository.ts

@@ -16,7 +16,16 @@ import { AppRestartEvent, ArgsOf, EventRepository } from 'src/repositories/event
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { handlePromiseError } from 'src/utils/misc';
 
-export const serverEvents = ['ConfigUpdate', 'AppRestart'] as const;
+export const serverEvents = [
+  'ConfigUpdate',
+  'AppRestart',
+  'HlsSegmentRequest',
+  'HlsSegmentResult',
+  'HlsHeartbeat',
+  'HlsSessionRequest',
+  'HlsSessionResult',
+  'HlsSessionEnd',
+] as const;
 export type ServerEvents = (typeof serverEvents)[number];
 
 export interface ClientEventMap {

server/src/schema/enums.ts

@@ -1,12 +1,5 @@
 import { registerEnum } from '@immich/sql-tools';
-import {
+import { AlbumUserRole, AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType, VideoCodec } from 'src/enum';
-  AlbumUserRole,
-  AssetStatus,
-  AssetVisibility,
-  ChecksumAlgorithm,
-  SourceType,
-  VideoSegmentCodec,
-} from 'src/enum';
 
 export const album_user_role_enum = registerEnum({
   name: 'album_user_role_enum',
@@ -35,5 +28,5 @@ export const asset_checksum_algorithm_enum = registerEnum({
 
 export const video_stream_variant_codec_enum = registerEnum({
   name: 'video_stream_variant_codec_enum',
-  values: Object.values(VideoSegmentCodec),
+  values: [VideoCodec.Av1, VideoCodec.Hevc, VideoCodec.H264],
 });

server/src/services/hls.service.spec.ts

@@ -0,0 +1,322 @@
+import { BadRequestException, NotFoundException } from '@nestjs/common';
+import { TranscodeHardwareAcceleration } from 'src/enum';
+import { HlsService } from 'src/services/hls.service';
+import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub';
+import { factory } from 'test/small.factory';
+import { newTestService, ServiceMocks } from 'test/utils';
+
+// EXTINF values come from FFmpeg's playlist to enforce an exact match
+const eiffelExpectedMediaPlaylist = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-TARGETDURATION:2
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="init.mp4"
+#EXTINF:2.007222,
+seg_0.m4s
+#EXTINF:2.007222,
+seg_1.m4s
+#EXTINF:2.007222,
+seg_2.m4s
+#EXTINF:2.007222,
+seg_3.m4s
+#EXTINF:2.007222,
+seg_4.m4s
+#EXTINF:2.007222,
+seg_5.m4s
+#EXTINF:2.007222,
+seg_6.m4s
+#EXTINF:2.007222,
+seg_7.m4s
+#EXTINF:2.007222,
+seg_8.m4s
+#EXTINF:2.007222,
+seg_9.m4s
+#EXTINF:2.007222,
+seg_10.m4s
+#EXTINF:0.281011,
+seg_11.m4s
+#EXT-X-ENDLIST
+`;
+
+const waterfallExpectedMediaPlaylist = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-TARGETDURATION:2
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="init.mp4"
+#EXTINF:2.011405,
+seg_0.m4s
+#EXTINF:2.011405,
+seg_1.m4s
+#EXTINF:2.011405,
+seg_2.m4s
+#EXTINF:2.011405,
+seg_3.m4s
+#EXTINF:2.011405,
+seg_4.m4s
+#EXTINF:0.301711,
+seg_5.m4s
+#EXT-X-ENDLIST
+`;
+
+const trainExpectedMediaPlaylist = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-TARGETDURATION:2
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MAP:URI="init.mp4"
+#EXTINF:2.000000,
+seg_0.m4s
+#EXTINF:2.000000,
+seg_1.m4s
+#EXTINF:2.000000,
+seg_2.m4s
+#EXTINF:2.000000,
+seg_3.m4s
+#EXTINF:2.000000,
+seg_4.m4s
+#EXTINF:2.000000,
+seg_5.m4s
+#EXTINF:2.000000,
+seg_6.m4s
+#EXTINF:2.000000,
+seg_7.m4s
+#EXTINF:2.000000,
+seg_8.m4s
+#EXTINF:2.000000,
+seg_9.m4s
+#EXTINF:1.733333,
+seg_10.m4s
+#EXT-X-ENDLIST
+`;
+
+const sessionId = '00000000-0000-0000-0000-000000000000';
+
+const eiffelExpectedMasterDisabled = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/0/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/1/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/2/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/3/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/4/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/5/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/6/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/7/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/8/playlist.m3u8
+`;
+
+const eiffelExpectedMasterRkmpp = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/1/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/2/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/4/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/5/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/7/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
+${sessionId}/8/playlist.m3u8
+`;
+
+const waterfallExpectedMasterDisabled = `#EXTM3U
+#EXT-X-VERSION:6
+#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/0/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/1/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/2/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/3/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/4/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/5/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/6/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/7/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/8/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=7000000,RESOLUTION=1440x2560,CODECS="av01.0.12M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/9/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1440x2560,CODECS="hvc1.2.4.L150.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/10/playlist.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=16000000,RESOLUTION=1440x2560,CODECS="avc1.640032,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
+${sessionId}/11/playlist.m3u8
+`;
+
+describe(HlsService.name, () => {
+  let sut: HlsService;
+  let mocks: ServiceMocks;
+
+  beforeEach(() => {
+    ({ sut, mocks } = newTestService(HlsService));
+  });
+
+  describe('getMasterPlaylist', () => {
+    const auth = factory.auth();
+    const assetId = 'asset-1';
+
+    const setup = (asset: typeof eiffelTower | typeof waterfall, accel: TranscodeHardwareAcceleration) => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true }, accel } });
+      mocks.videoStream.getForMasterPlaylist.mockResolvedValue(asset);
+      mocks.crypto.randomUUID.mockReturnValue(sessionId);
+      mocks.websocket.serverSend.mockImplementation((event, ...rest) => {
+        if (event === 'HlsSessionRequest') {
+          const { sessionId: id } = rest[0] as { sessionId: string };
+          queueMicrotask(() => sut.onSessionResult({ sessionId: id }));
+        }
+      });
+    };
+
+    it('returns master playlist for eiffel-tower (1080p portrait, no acceleration)', async () => {
+      setup(eiffelTower, TranscodeHardwareAcceleration.Disabled);
+      await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterDisabled);
+    });
+
+    it('returns master playlist for eiffel-tower with RKMPP (no AV1 variants)', async () => {
+      setup(eiffelTower, TranscodeHardwareAcceleration.Rkmpp);
+      await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterRkmpp);
+    });
+
+    it('returns master playlist for waterfall (4K landscape) with no acceleration', async () => {
+      setup(waterfall, TranscodeHardwareAcceleration.Disabled);
+      await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(waterfallExpectedMasterDisabled);
+    });
+
+    it('throws BadRequestException when realtime transcoding is disabled', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: false } } });
+      await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(BadRequestException);
+    });
+
+    it('throws NotFoundException when asset is not yet ready for streaming', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true } } });
+      await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(NotFoundException);
+    });
+  });
+
+  describe('getMediaPlaylist', () => {
+    const auth = factory.auth();
+    const assetId = 'asset-1';
+    const fixtures = [
+      { data: eiffelTower, playlist: eiffelExpectedMediaPlaylist },
+      { data: waterfall, playlist: waterfallExpectedMediaPlaylist },
+      { data: train, playlist: trainExpectedMediaPlaylist },
+    ];
+
+    it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data);
+      await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist);
+    });
+
+    it('throws NotFoundException when the session/asset cannot be loaded', async () => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException);
+    });
+  });
+
+  describe('getSegment', () => {
+    const auth = factory.auth();
+    const assetId = 'asset-1';
+    const variantIndex = 0;
+
+    beforeEach(() => {
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);
+      mocks.storage.checkFileExists.mockResolvedValue(true);
+    });
+
+    it('emits HlsHeartbeat with segmentIndex 0 for the first init.mp4 request', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 0,
+      });
+    });
+
+    it('emits HlsHeartbeat with the parsed segment number for seg_K.m4s', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 5,
+      });
+    });
+
+    it('returns lastRequested + 1 for init.mp4 after a segment has been served', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
+      mocks.websocket.serverSend.mockClear();
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 6,
+      });
+    });
+
+    it('updates lastRequested on a backward-seek segment request', async () => {
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_3.m4s');
+      mocks.websocket.serverSend.mockClear();
+      await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId,
+        variantIndex,
+        segmentIndex: 4,
+      });
+    });
+
+    it('tracks segment state per session independently', async () => {
+      await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'seg_5.m4s');
+      await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'seg_2.m4s');
+      mocks.websocket.serverSend.mockClear();
+      await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'init.mp4');
+      await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'init.mp4');
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId: 'session-a',
+        variantIndex,
+        segmentIndex: 6,
+      });
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
+        sessionId: 'session-b',
+        variantIndex,
+        segmentIndex: 3,
+      });
+    });
+
+    it('throws NotFoundException when the session does not exist', async () => {
+      await expect(sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4')).rejects.toBeInstanceOf(
+        NotFoundException,
+      );
+    });
+  });
+
+  describe('endSession', () => {
+    it('emits HlsSessionEnd', async () => {
+      const auth = factory.auth();
+      const assetId = 'asset-1';
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
+      await sut.endSession(auth, assetId, sessionId);
+      expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSessionEnd', { sessionId });
+    });
+  });
+});

server/src/services/hls.service.ts

@@ -0,0 +1,202 @@
+import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
+import { constants } from 'node:fs';
+import { join } from 'node:path';
+import {
+  HLS_SEGMENT_DURATION,
+  HLS_VARIANTS,
+  HLS_VERSION,
+  SEGMENT_FILENAME_REGEX,
+  SUPPORTED_HWA_CODECS,
+} from 'src/constants';
+import { StorageCore } from 'src/cores/storage.core';
+import { OnEvent } from 'src/decorators';
+import { AuthDto } from 'src/dtos/auth.dto';
+import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
+import { CacheControl, ImmichWorker, Permission } from 'src/enum';
+import { ArgOf } from 'src/repositories/event.repository';
+import { BaseService } from 'src/services/base.service';
+import { VideoPacketInfo, VideoStreamInfo } from 'src/types';
+import { PendingEvents } from 'src/utils/event';
+import { ImmichFileResponse } from 'src/utils/file';
+import { getOutputSize } from 'src/utils/media';
+
+type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo };
+
+@Injectable()
+export class HlsService extends BaseService {
+  private pendingSegments = new PendingEvents<'HlsSegmentResult'>(15_000);
+  private pendingSessions = new PendingEvents<'HlsSessionResult'>(5000);
+  private sessions = new Map<string, { lastRequestedSegment: number | null }>();
+
+  @OnEvent({ name: 'HlsSessionResult', server: true, workers: [ImmichWorker.Api] })
+  onSessionResult(event: ArgOf<'HlsSessionResult'>) {
+    this.pendingSessions.complete(event.sessionId, event);
+    if (event.error) {
+      this.sessions.delete(event.sessionId);
+      this.pendingSegments.rejectByPrefix(`${event.sessionId}:`, event.error);
+    }
+  }
+
+  @OnEvent({ name: 'HlsSessionEnd', server: true, workers: [ImmichWorker.Api] })
+  onSessionEnd({ sessionId }: ArgOf<'HlsSessionEnd'>) {
+    this.sessions.delete(sessionId);
+    this.pendingSegments.rejectByPrefix(`${sessionId}:`, 'Session ended');
+  }
+
+  @OnEvent({ name: 'HlsSegmentResult', server: true, workers: [ImmichWorker.Api] })
+  onSegmentResult(event: ArgOf<'HlsSegmentResult'>) {
+    this.pendingSegments.complete(this.getSegmentKey(event), event);
+  }
+
+  async getMasterPlaylist(auth: AuthDto, assetId: string) {
+    await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
+    const { ffmpeg } = await this.getConfig({ withCache: true });
+    if (!ffmpeg.realtime.enabled) {
+      throw new BadRequestException('Real-time transcoding is not enabled');
+    }
+
+    const asset = await this.videoStreamRepository.getForMasterPlaylist(assetId);
+    if (!asset) {
+      throw new NotFoundException('Asset is not yet ready for streaming');
+    }
+
+    const sessionId = this.cryptoRepository.randomUUID();
+    this.websocketRepository.serverSend('HlsSessionRequest', { sessionId, assetId, ownerId: auth.user.id });
+    await this.pendingSessions.wait(sessionId);
+    this.sessions.set(sessionId, { lastRequestedSegment: null });
+
+    return this.generateMasterPlaylist(sessionId, ffmpeg, asset);
+  }
+
+  async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) {
+    await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
+
+    const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId);
+    if (!asset) {
+      throw new NotFoundException('Asset not found or not yet ready for streaming');
+    }
+
+    return this.generateMediaPlaylist(asset);
+  }
+
+  async getSegment(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, filename: string) {
+    const t0 = performance.now();
+    await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
+    const t1 = performance.now();
+
+    const session = await this.videoStreamRepository.getSession(sessionId);
+    if (!session) {
+      throw new NotFoundException('Session not found');
+    }
+    const t2 = performance.now();
+
+    const variantDir = StorageCore.getHlsVariantFolder({ ownerId: auth.user.id, sessionId, variantIndex });
+    const path = join(variantDir, filename);
+    const response = new ImmichFileResponse({
+      path,
+      contentType: 'video/mp4',
+      cacheControl: CacheControl.PrivateWithCache,
+    });
+
+    const segmentIndex = this.getSegmentIndex(sessionId, filename);
+    this.websocketRepository.serverSend('HlsHeartbeat', { sessionId, variantIndex, segmentIndex });
+
+    if (await this.storageRepository.checkFileExists(path, constants.R_OK)) {
+      this.logger.log(
+        `[TIMING] getSegment(cached) session=${sessionId} variant=${variantIndex} file=${filename} ` +
+          `auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` +
+          `total=${(performance.now() - t0).toFixed(1)}ms`,
+      );
+      return response;
+    }
+    const t3 = performance.now();
+
+    this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex });
+    await this.pendingSegments.wait(this.getSegmentKey({ sessionId, variantIndex, segmentIndex }));
+    const t4 = performance.now();
+
+    this.logger.log(
+      `[TIMING] getSegment(wait) session=${sessionId} variant=${variantIndex} file=${filename} ` +
+        `auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` +
+        `existsCheck=${(t3 - t2).toFixed(1)}ms wait=${(t4 - t3).toFixed(1)}ms ` +
+        `total=${(t4 - t0).toFixed(1)}ms`,
+    );
+
+    return response;
+  }
+
+  async endSession(auth: AuthDto, assetId: string, sessionId: string): Promise<void> {
+    await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
+
+    this.websocketRepository.serverSend('HlsSessionEnd', { sessionId });
+  }
+
+  private generateMasterPlaylist(sessionId: string, ffmpeg: SystemConfigFFmpegDto, asset: AssetWithStreamInfo) {
+    const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3);
+    const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width);
+    const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`];
+    for (let i = 0; i < HLS_VARIANTS.length; i++) {
+      const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i];
+      if (resolution > sourceResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) {
+        continue;
+      }
+      const { width, height } = getOutputSize(asset.videoStream, resolution);
+      lines.push(
+        `#EXT-X-STREAM-INF:BANDWIDTH=${bitrate},RESOLUTION=${width}x${height},CODECS="${codecString},mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=${fps}`,
+        `${sessionId}/${i}/playlist.m3u8`,
+      );
+    }
+    lines.push('');
+
+    if (lines.length === 3) {
+      throw new NotFoundException('No supported variants for this video');
+    }
+
+    return lines.join('\n');
+  }
+
+  private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) {
+    const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration;
+    const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps);
+    const fullSegmentDuration = framesPerSegment / fps;
+    const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment);
+    const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1);
+    const lastSegmentDuration = lastSegmentFrames / fps;
+
+    const lines = [
+      '#EXTM3U',
+      `#EXT-X-VERSION:${HLS_VERSION}`,
+      `#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`,
+      '#EXT-X-MEDIA-SEQUENCE:0',
+      '#EXT-X-PLAYLIST-TYPE:VOD',
+      '#EXT-X-MAP:URI="init.mp4"',
+    ];
+
+    for (let i = 0; i < segmentCount - 1; i++) {
+      lines.push(`#EXTINF:${fullSegmentDuration.toFixed(6)},`, `seg_${i}.m4s`);
+    }
+    lines.push(`#EXTINF:${lastSegmentDuration.toFixed(6)},`, `seg_${segmentCount - 1}.m4s`, '#EXT-X-ENDLIST', '');
+
+    return lines.join('\n');
+  }
+
+  private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) {
+    return `${sessionId}:${variantIndex}:${segmentIndex}`;
+  }
+
+  private getSegmentIndex(sessionId: string, filename: string) {
+    const existing = this.sessions.get(sessionId);
+    let segmentIndex: number;
+    if (filename.endsWith('.mp4')) {
+      segmentIndex = (existing?.lastRequestedSegment ?? -1) + 1;
+    } else {
+      segmentIndex = Number.parseInt(SEGMENT_FILENAME_REGEX.exec(filename)![1]);
+      if (existing) {
+        existing.lastRequestedSegment = segmentIndex;
+      } else {
+        this.sessions.set(sessionId, { lastRequestedSegment: segmentIndex });
+      }
+    }
+    return segmentIndex;
+  }
+}

server/src/services/index.ts

@@ -11,6 +11,7 @@ import { DatabaseBackupService } from 'src/services/database-backup.service';
 import { DatabaseService } from 'src/services/database.service';
 import { DownloadService } from 'src/services/download.service';
 import { DuplicateService } from 'src/services/duplicate.service';
+import { HlsService } from 'src/services/hls.service';
 import { JobService } from 'src/services/job.service';
 import { LibraryService } from 'src/services/library.service';
 import { MaintenanceService } from 'src/services/maintenance.service';
@@ -60,6 +61,7 @@ export const services = [
   DatabaseService,
   DownloadService,
   DuplicateService,
+  HlsService,
   JobService,
   LibraryService,
   MaintenanceService,

server/src/services/system-config.service.spec.ts

@@ -72,6 +72,9 @@ const updatedConfig = Object.freeze<SystemConfig>({
     accel: TranscodeHardwareAcceleration.Disabled,
     accelDecode: false,
     tonemap: ToneMapping.Hable,
+    realtime: {
+      enabled: false,
+    },
   },
   logging: {
     enabled: true,

server/src/utils/event.ts

@@ -0,0 +1,44 @@
+import { ArgOf, EmitEvent } from 'src/repositories/event.repository';
+
+export class PendingEvents<T extends { [T in EmitEvent]: ArgOf<T> extends { error?: string } ? T : never }[EmitEvent]> {
+  private pending = new Map<string, { completers: PromiseWithResolvers<ArgOf<T>>[]; timeout: NodeJS.Timeout }>();
+
+  constructor(private timeoutMs: number) {}
+
+  wait(key: string): Promise<ArgOf<T>> {
+    const completer = Promise.withResolvers<ArgOf<T>>();
+    const existing = this.pending.get(key);
+    if (existing) {
+      existing.completers.push(completer);
+      return completer.promise;
+    }
+
+    const timeout = setTimeout(() => this.complete(key, { error: 'Request timed out' }), this.timeoutMs);
+    this.pending.set(key, { completers: [completer], timeout });
+    return completer.promise;
+  }
+
+  complete(key: string, value: ArgOf<T> | { error: string }) {
+    const pending = this.pending.get(key);
+    if (!pending) {
+      return;
+    }
+    clearTimeout(pending.timeout);
+    this.pending.delete(key);
+    for (const completer of pending.completers) {
+      if ('error' in value) {
+        completer.reject(new Error(value.error));
+      } else {
+        completer.resolve(value);
+      }
+    }
+  }
+
+  rejectByPrefix(prefix: string, error: string) {
+    for (const key of this.pending.keys()) {
+      if (key.startsWith(prefix)) {
+        this.complete(key, { error });
+      }
+    }
+  }
+}

server/test/fixtures/media.stub.ts

@@ -597,7 +597,7 @@ export const train = {
   packets: {
     totalDuration: 12_290,
     packetCount: 1229,
-    outputFrames: 1303,
+    outputFrames: 1304,
     keyframePts: [
       0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210, 10_811,
       11_411, 12_062, 12_703,

server/tsconfig.json

@@ -8,9 +8,9 @@
     "experimentalDecorators": true,
     "allowSyntheticDefaultImports": true,
     "resolveJsonModule": true,
-    "target": "es2022",
+    "target": "es2024",
     "moduleResolution": "node16",
-    "lib": ["dom", "es2023"],
+    "lib": ["dom", "es2024"],
     "sourceMap": true,
     "outDir": "./dist",
     "incremental": true,