hls player

pnpm-lock.yaml

@@ -789,6 +789,12 @@ importers:
       happy-dom:
         specifier: ^20.0.0
         version: 20.9.0
+      hls-video-element:
+        specifier: ^1.5.11
+        version: 1.5.11
+      hls.js:
+        specifier: ^1.6.16
+        version: 1.6.16
       intl-messageformat:
         specifier: ^11.0.0
         version: 11.2.1
@@ -6739,6 +6745,9 @@ packages:
   csstype@3.2.3:
     resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
 
+  custom-media-element@1.4.6:
+    resolution: {integrity: sha512-/HRYqJOa1ob5ik4q7FIJVYxTJCFs/FL3+cQPAJjUf2uiqrDEzbTgB315gQ2rG8oK3w094W9m5tcB8S5Qah+caA==}
+
   cytoscape-cose-bilkent@4.1.0:
     resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==}
     peerDependencies:
@@ -8060,6 +8069,12 @@ packages:
   history@4.10.1:
     resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==}
 
+  hls-video-element@1.5.11:
+    resolution: {integrity: sha512-tJJ65/52CDxj8XFyIve6zT9nVVdUIc6mqvKR25X0ycPKHk07rpjp4xxVteeCefDUBSf/tFLhlICFmn3KWj37xA==}
+
+  hls.js@1.6.16:
+    resolution: {integrity: sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA==}
+
   hogan.js@3.0.2:
     resolution: {integrity: sha512-RqGs4wavGYJWE07t35JQccByczmNUXQT0E12ZYV1VKYu5UiAU9lsos/yBAcf840+zrUQQxgVduCR5/B8nNtibg==}
     hasBin: true
@@ -9091,6 +9106,9 @@ packages:
   media-chrome@4.19.0:
     resolution: {integrity: sha512-HWhDTwts+BSbdPkkB1VsJXp5kvL0IxY7xFT5tBwliM2+89kTPVTnHnev+9it2f9PweANjT/C8/C/S0PW9oyZbA==}
 
+  media-tracks@0.3.5:
+    resolution: {integrity: sha512-l54rkKXlLBt3ob3zOLWHcnjvwUmX5bNEZ70igyapOZZC9imzqBmq1oz8p2roiV04KhjblFIi2hetLPF1oYVLRA==}
+
   media-typer@0.3.0:
     resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
     engines: {node: '>= 0.6'}
@@ -19347,6 +19365,8 @@ snapshots:
 
   csstype@3.2.3: {}
 
+  custom-media-element@1.4.6: {}
+
   cytoscape-cose-bilkent@4.1.0(cytoscape@3.33.1):
     dependencies:
       cose-base: 1.0.3
@@ -21018,6 +21038,14 @@ snapshots:
       tiny-warning: 1.0.3
       value-equal: 1.0.1
 
+  hls-video-element@1.5.11:
+    dependencies:
+      custom-media-element: 1.4.6
+      hls.js: 1.6.16
+      media-tracks: 0.3.5
+
+  hls.js@1.6.16: {}
+
   hogan.js@3.0.2:
     dependencies:
       mkdirp: 0.3.0
@@ -22188,6 +22216,8 @@ snapshots:
     transitivePeerDependencies:
       - react
 
+  media-tracks@0.3.5: {}
+
   media-typer@0.3.0: {}
 
   media-typer@1.1.0: {}

web/package.json

@@ -46,6 +46,8 @@
     "geojson": "^0.5.0",
     "handlebars": "^4.7.8",
     "happy-dom": "^20.0.0",
+    "hls-video-element": "^1.5.11",
+    "hls.js": "^1.6.16",
     "intl-messageformat": "^11.0.0",
     "justified-layout": "^4.1.0",
     "lodash-es": "^4.17.21",

web/src/lib/components/asset-viewer/VideoNativeViewer.svelte

@@ -5,7 +5,7 @@
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
   import { castManager } from '$lib/managers/cast-manager.svelte';
   import { autoPlayVideo, loopVideo as loopVideoPreference } from '$lib/stores/preferences.store';
-  import { getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
+  import { getAssetHlsSessionUrl, getAssetHlsUrl, getAssetMediaUrl, getAssetPlaybackUrl } from '$lib/utils';
   import { AssetMediaSize, type AssetResponseDto } from '@immich/sdk';
   import { Icon, LoadingSpinner } from '@immich/ui';
   import {
@@ -21,6 +21,9 @@
     mdiVolumeMedium,
     mdiVolumeMute,
   } from '@mdi/js';
+  import Hls, { AbrController, type HlsConfig } from 'hls.js';
+  import 'hls-video-element';
+  import type HlsVideoElement from 'hls-video-element';
   import 'media-chrome/media-control-bar';
   import 'media-chrome/media-controller';
   import 'media-chrome/media-fullscreen-button';
@@ -38,6 +41,7 @@
   import { useSwipe, type SwipeCustomEvent } from 'svelte-gestures';
   import { t } from 'svelte-i18n';
   import { fade } from 'svelte/transition';
+  import { featureFlagsManager } from '$lib/managers/feature-flags-manager.svelte';
 
   interface Props {
     asset: AssetResponseDto;
@@ -69,14 +73,105 @@
 
   let videoPlayer: HTMLVideoElement | undefined = $state();
   let isLoading = $state(true);
-  let assetFileUrl = $derived(
+  let assetFileUrl = $derived.by(() => {
-    playOriginalVideo
+    if (featureFlagsManager.value.realtimeTranscoding) {
-      ? getAssetMediaUrl({ id: assetId, size: AssetMediaSize.Original, cacheKey })
+      return getAssetHlsUrl(assetId);
-      : getAssetPlaybackUrl({ id: assetId, cacheKey }),
+    }
-  );
+
+    if (playOriginalVideo) {
+      return getAssetMediaUrl({ id: assetId, size: AssetMediaSize.Original, cacheKey });
+    }
+
+    return getAssetPlaybackUrl({ id: assetId, cacheKey });
+  });
   const aspectRatio = $derived(asset.width && asset.height ? `${asset.width} / ${asset.height}` : undefined);
   let showVideo = $state(false);
   let hasFocused = $state(false);
+  let activeSession: { assetId: string; id: string } | undefined;
+  let rebuildCount = 0;
+
+  const MAX_REBUILDS = 1;
+  const SESSION_ID_REGEX = /\/video\/stream\/([0-9a-f-]{36})\//;
+
+  // hls.js can abandon fetching an in-flight fragment if it thinks it'll take too long, in which case
+  // it emergency switches to a different variant. This extends the delay even further due to
+  // cold starting another transcode, so let the fragment finish and have steady ABR decide the next level.
+  class NoAbandonAbrController extends AbrController {
+    protected override onFragLoading() {}
+  }
+
+  const hlsConfig: Partial<HlsConfig> = {
+    abrController: NoAbandonAbrController,
+    highBufferWatchdogPeriod: 10,
+    detectStallWithCurrentTimeMs: 10_000,
+    maxBufferHole: 0.5,
+    maxBufferLength: 30,
+    maxMaxBufferLength: 60,
+    fragLoadPolicy: {
+      default: {
+        maxTimeToFirstByteMs: 30_000,
+        maxLoadTimeMs: 60_000,
+        timeoutRetry: { maxNumRetry: 5, retryDelayMs: 100, maxRetryDelayMs: 0 },
+        errorRetry: { maxNumRetry: 3, retryDelayMs: 1000, maxRetryDelayMs: 8000 },
+      },
+    },
+  };
+
+  const releaseSession = () => {
+    const session = activeSession;
+    if (!session) {
+      return;
+    }
+    activeSession = undefined;
+    const url = getAssetHlsSessionUrl(session.assetId, session.id);
+    void fetch(url, { method: 'DELETE' }).catch(() => console.warn('Failed to release HLS session', session));
+  };
+
+  const isHlsElement = (el: HTMLVideoElement | undefined): el is HlsVideoElement => {
+    return el?.tagName === 'HLS-VIDEO';
+  };
+
+  const wireHlsListeners = (el: HlsVideoElement, assetId: string, resumeTime?: number) => {
+    const api = el.api;
+    if (!api) {
+      return;
+    }
+
+    api.on(Hls.Events.MANIFEST_PARSED, () => {
+      const id = api.levels[0]?.url[0]?.match(SESSION_ID_REGEX)?.[1];
+      if (id) {
+        activeSession = { assetId, id };
+      }
+    });
+
+    api.on(Hls.Events.FRAG_LOADED, () => (rebuildCount = 0));
+
+    api.on(Hls.Events.ERROR, (_, data) => {
+      console.error('HLS error', JSON.stringify(data));
+      // 404 on a fragment can mean the server-side session has expired. Refetch
+      // master for a new session, but give up if it still 404s.
+      if (
+        !data.fatal ||
+        data.details !== Hls.ErrorDetails.FRAG_LOAD_ERROR ||
+        data.response?.code !== 404 ||
+        rebuildCount++ >= MAX_REBUILDS
+      ) {
+        return;
+      }
+      activeSession = undefined;
+      resumeTime = el.currentTime;
+      // Re-setting src triggers attributeChangedCallback → load(), which
+      // destroys the old Hls and instantiates a new one with our config.
+      const url = el.src;
+      el.removeAttribute('src');
+      el.setAttribute('src', url);
+      queueMicrotask(() => wireHlsListeners(el, assetId, resumeTime));
+    });
+
+    if (resumeTime) {
+      el.addEventListener('loadedmetadata', () => (el.currentTime = resumeTime!), { once: true });
+    }
+  };
 
   onMount(() => {
     showVideo = true;
@@ -84,10 +179,31 @@
 
   $effect(() => {
     // reactive on `assetFileUrl` changes
-    if (assetFileUrl) {
+    if (videoPlayer && assetFileUrl) {
       hasFocused = false;
-      videoPlayer?.load();
+      rebuildCount = 0;
+      releaseSession();
+      if (isHlsElement(videoPlayer)) {
+        videoPlayer.config = hlsConfig;
+        videoPlayer.src = assetFileUrl;
+        const el = videoPlayer;
+        queueMicrotask(() => wireHlsListeners(el, assetId));
+      } else {
+        videoPlayer.load();
+      }
     }
+    return releaseSession;
+  });
+
+  const onPagehide = (event: PageTransitionEvent) => {
+    if (!event.persisted) {
+      releaseSession();
+    }
+  };
+
+  $effect(() => {
+    window.addEventListener('pagehide', onPagehide);
+    return () => window.removeEventListener('pagehide', onPagehide);
   });
 
   onDestroy(() => {
@@ -171,27 +287,49 @@
         style:aspect-ratio={aspectRatio}
         defaultduration={asset.duration! / 1000}
       >
-        <video
+        {#if featureFlagsManager.value.realtimeTranscoding}
-          bind:this={videoPlayer}
+          <hls-video
-          slot="media"
+            bind:this={videoPlayer}
-          loop={$loopVideoPreference && loopVideo}
+            slot="media"
-          autoplay={$autoPlayVideo}
+            loop={$loopVideoPreference && loopVideo}
-          disablePictureInPicture
+            autoplay={$autoPlayVideo}
-          playsinline
+            disablePictureInPicture
-          {...useSwipe(onSwipe)}
+            playsinline
-          class="h-full object-contain"
+            {...useSwipe(onSwipe)}
-          oncanplay={(e) => handleCanPlay(e.currentTarget)}
+            class="h-full object-contain"
-          onended={onVideoEnded}
+            oncanplay={(e: Event) => handleCanPlay(e.currentTarget as HTMLVideoElement)}
-          onplaying={(e) => {
+            onended={onVideoEnded}
-            if (!hasFocused) {
+            onplaying={(e: Event) => {
-              e.currentTarget.focus();
+              if (!hasFocused) {
-              hasFocused = true;
+                (e.currentTarget as HTMLElement).focus();
-            }
+                hasFocused = true;
-          }}
+              }
-          onclose={onClose}
+            }}
-          poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
+            onclose={onClose}
-          src={assetFileUrl}
+            poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
-        ></video>
+          ></hls-video>
+        {:else}
+          <video
+            bind:this={videoPlayer}
+            slot="media"
+            loop={$loopVideoPreference && loopVideo}
+            autoplay={$autoPlayVideo}
+            disablePictureInPicture
+            playsinline
+            {...useSwipe(onSwipe)}
+            class="h-full object-contain"
+            oncanplay={(e) => handleCanPlay(e.currentTarget)}
+            onended={onVideoEnded}
+            onplaying={(e) => {
+              if (!hasFocused) {
+                e.currentTarget.focus();
+                hasFocused = true;
+              }
+            }}
+            onclose={onClose}
+            poster={getAssetMediaUrl({ id: asset.id, size: AssetMediaSize.Preview, cacheKey })}
+          ></video>
+        {/if}
 
         {#if extendedControls}
           <media-settings-menu hidden anchor="auto" class="w-3xs rounded-xl border border-light-300 shadow-sm">
@@ -247,7 +385,7 @@
         </div>
       {/if}
 
-      {#if assetViewerManager.isFaceEditMode}
+      {#if assetViewerManager.isFaceEditMode && videoPlayer}
         <FaceEditor htmlElement={videoPlayer} {containerWidth} {containerHeight} {assetId} />
       {/if}
     {/if}

web/src/lib/utils.ts

@@ -243,6 +243,14 @@ export const getAssetPlaybackUrl = (options: AssetUrlOptions) => {
   return createUrl(getAssetPlaybackPath(id), { ...authManager.params, c });
 };
 
+export const getAssetHlsUrl = (id: string) => {
+  return createUrl(`/assets/${id}/video/stream/main.m3u8`, authManager.params);
+};
+
+export const getAssetHlsSessionUrl = (id: string, sessionId: string) => {
+  return createUrl(`/assets/${id}/video/stream/${sessionId}`, authManager.params);
+};
+
 export const getProfileImageUrl = (user: UserResponseDto) =>
   createUrl(getUserProfileImagePath(user.id), { updatedAt: user.profileChangedAt });