transcoding service

server/src/utils/database.ts

@@ -71,10 +71,13 @@ export const removeUndefinedKeys = <T extends object>(update: T, template: unkno
 };
 
 export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum';
+export const VIDEO_STREAM_SESSION_PK_CONSTRAINT = 'video_stream_session_pkey';
 
-export const isAssetChecksumConstraint = (error: unknown) => {
-  return (error as PostgresError)?.constraint_name === 'UQ_assets_owner_checksum';
-};
+export const isAssetChecksumConstraint = (error: unknown) =>
+  (error as PostgresError)?.constraint_name === ASSET_CHECKSUM_CONSTRAINT;
+
+export const isVideoStreamSessionPkConstraint = (error: unknown) =>
+  (error as PostgresError)?.constraint_name === VIDEO_STREAM_SESSION_PK_CONSTRAINT;
 
 export function withDefaultVisibility<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
   return qb.where('asset.visibility', 'in', [sql.lit(AssetVisibility.Archive), sql.lit(AssetVisibility.Timeline)]);

server/src/utils/media.ts

@@ -1,4 +1,4 @@
-import { AUDIO_ENCODER } from 'src/constants';
+import { AUDIO_ENCODER, SUPPORTED_HWA_CODECS } from 'src/constants';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
 import {
   ColorMatrix,
@@ -13,38 +13,56 @@ import {
 import {
   AudioStreamInfo,
   BitrateDistribution,
+  HlsCommandOptions,
   TranscodeCommand,
-  VideoCodecHWConfig,
   VideoCodecSWConfig,
   VideoFormat,
   VideoInterfaces,
   VideoStreamInfo,
+  VideoTuning,
 } from 'src/types';
 
+export const isVideoRotated = (videoStream: VideoStreamInfo): boolean => Math.abs(videoStream.rotation) === 90;
+
+export const isVideoVertical = (videoStream: VideoStreamInfo): boolean =>
+  videoStream.height > videoStream.width || isVideoRotated(videoStream);
+
+export const getOutputSize = (videoStream: VideoStreamInfo, targetRes: number) => {
+  const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
+  let larger = Math.round(targetRes * factor);
+  if (larger % 2 !== 0) {
+    larger -= 1;
+  }
+  return isVideoVertical(videoStream) ? { width: targetRes, height: larger } : { width: larger, height: targetRes };
+};
+
 export class BaseConfig implements VideoCodecSWConfig {
   readonly presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
-  protected constructor(protected config: SystemConfigFFmpegDto) {}
+  protected constructor(
+    protected config: SystemConfigFFmpegDto,
+    protected tune: VideoTuning = { strictGop: false, lowLatency: false },
+  ) {}
 
-  static create(config: SystemConfigFFmpegDto, interfaces: VideoInterfaces): VideoCodecSWConfig {
+  static create(config: SystemConfigFFmpegDto, interfaces: VideoInterfaces, tune?: VideoTuning) {
     if (config.accel === TranscodeHardwareAcceleration.Disabled) {
-      return this.getSWCodecConfig(config);
+      return this.getSWCodecConfig(config, tune);
     }
-    return this.getHWCodecConfig(config, interfaces);
+    return this.getHWCodecConfig(config, interfaces, tune);
   }
 
-  private static getSWCodecConfig(config: SystemConfigFFmpegDto) {
+  private static getSWCodecConfig(config: SystemConfigFFmpegDto, tune?: VideoTuning): VideoCodecSWConfig {
     switch (config.targetVideoCodec) {
       case VideoCodec.H264: {
-        return new H264Config(config);
+        return new H264Config(config, tune);
       }
       case VideoCodec.Hevc: {
-        return new HEVCConfig(config);
+        return new HEVCConfig(config, tune);
       }
       case VideoCodec.Vp9: {
-        return new VP9Config(config);
+        return new VP9Config(config, tune);
       }
       case VideoCodec.Av1: {
-        return new AV1Config(config);
+        return new AV1Config(config, tune);
       }
       default: {
         throw new Error(`Codec '${config.targetVideoCodec}' is unsupported`);
@@ -52,72 +70,126 @@ export class BaseConfig implements VideoCodecSWConfig {
     }
   }
 
-  private static getHWCodecConfig(config: SystemConfigFFmpegDto, interfaces: VideoInterfaces) {
-    let handler: VideoCodecHWConfig;
+  private static getHWCodecConfig(config: SystemConfigFFmpegDto, interfaces: VideoInterfaces, tune?: VideoTuning) {
+    if (!SUPPORTED_HWA_CODECS[config.accel].includes(config.targetVideoCodec)) {
+      throw new Error(
+        `${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${SUPPORTED_HWA_CODECS[config.accel]}`,
+      );
+    }
+
+    let handler: VideoCodecSWConfig;
     switch (config.accel) {
       case TranscodeHardwareAcceleration.Nvenc: {
         handler = config.accelDecode
-          ? new NvencHwDecodeConfig(config, interfaces)
-          : new NvencSwDecodeConfig(config, interfaces);
+          ? new NvencHwDecodeConfig(config, interfaces, tune)
+          : new NvencSwDecodeConfig(config, interfaces, tune);
         break;
       }
       case TranscodeHardwareAcceleration.Qsv: {
         handler = config.accelDecode
-          ? new QsvHwDecodeConfig(config, interfaces)
-          : new QsvSwDecodeConfig(config, interfaces);
+          ? new QsvHwDecodeConfig(config, interfaces, tune)
+          : new QsvSwDecodeConfig(config, interfaces, tune);
         break;
       }
       case TranscodeHardwareAcceleration.Vaapi: {
         handler = config.accelDecode
-          ? new VaapiHwDecodeConfig(config, interfaces)
-          : new VaapiSwDecodeConfig(config, interfaces);
+          ? new VaapiHwDecodeConfig(config, interfaces, tune)
+          : new VaapiSwDecodeConfig(config, interfaces, tune);
         break;
       }
       case TranscodeHardwareAcceleration.Rkmpp: {
         handler = config.accelDecode
-          ? new RkmppHwDecodeConfig(config, interfaces)
-          : new RkmppSwDecodeConfig(config, interfaces);
+          ? new RkmppHwDecodeConfig(config, interfaces, tune)
+          : new RkmppSwDecodeConfig(config, interfaces, tune);
         break;
       }
       default: {
         throw new Error(`${config.accel.toUpperCase()} acceleration is unsupported`);
       }
     }
-    if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) {
-      throw new Error(
-        `${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`,
-      );
-    }
 
     return handler;
   }
 
-  getCommand(
-    target: TranscodeTarget,
-    videoStream: VideoStreamInfo,
-    audioStream?: AudioStreamInfo,
-    format?: VideoFormat,
-  ) {
+  getCommand(target: TranscodeTarget, video: VideoStreamInfo, audio?: AudioStreamInfo, format?: VideoFormat) {
     const options = {
-      inputOptions: this.getBaseInputOptions(videoStream, format),
-      outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v', 'verbose'],
+      inputOptions: this.getBaseInputOptions(video, format),
+      outputOptions: [
+        ...this.getBaseOutputOptions(target, video, audio),
+        ...this.getPresetOptions(),
+        ...this.getBitrateOptions(),
+        ...this.getEncoderOptions(),
+        '-movflags',
+        'faststart',
+        '-fps_mode',
+        'passthrough',
+        '-v',
+        'verbose',
+      ],
       twoPass: this.eligibleForTwoPass(),
-      progress: { frameCount: videoStream.frameCount, percentInterval: 5 },
+      progress: { frameCount: video.frameCount, percentInterval: 5 },
     } as TranscodeCommand;
     if ([TranscodeTarget.All, TranscodeTarget.Video].includes(target)) {
-      const filters = this.getFilterOptions(videoStream);
+      const filters = this.getFilterOptions(video);
       if (filters.length > 0) {
         options.outputOptions.push('-vf', filters.join(','));
       }
     }
 
-    options.outputOptions.push(
+    return options;
+  }
+
+  getHlsCommand(options: HlsCommandOptions, video: VideoStreamInfo, audio?: AudioStreamInfo) {
+    const args: string[] = this.getBaseInputOptions(video);
+    if (options.seekSeconds) {
+      args.push('-ss', String(options.seekSeconds));
+    }
+    args.push(
+      '-nostdin',
+      '-nostats',
+      '-i',
+      options.inputPath,
+      ...this.getBaseOutputOptions(options.target, video, audio),
       ...this.getPresetOptions(),
-      ...this.getOutputThreadOptions(),
       ...this.getBitrateOptions(),
+      ...this.getEncoderOptions(),
+      // -start_at_zero only applies meaningfully when seekSeconds is 0.
+      // Combined with -ss N -copyts, it would shift the first frame to t=0
+      // and break tfdt alignment with the playlist.
+      ...(options.seekSeconds ? [] : ['-start_at_zero']),
+      '-copyts',
+      '-r',
+      `${options.packetCount * options.timeBase}/${options.totalDuration}`,
+      '-avoid_negative_ts',
+      'disabled',
+      '-f',
+      'hls',
+      '-hls_time',
+      String(options.segmentDuration),
+      '-hls_list_size',
+      '0',
+      '-hls_segment_type',
+      'fmp4',
+      '-hls_fmp4_init_filename',
+      options.initFilename,
+      '-hls_segment_options',
+      'movflags=+frag_discont',
+      '-hls_flags',
+      'temp_file',
+      '-hls_segment_filename',
+      options.segmentFilename,
+      '-start_number',
+      String(options.startSegment),
     );
 
-    return options;
+    if ([TranscodeTarget.All, TranscodeTarget.Video].includes(options.target)) {
+      const filters = this.getFilterOptions(video);
+      if (filters.length > 0) {
+        args.push('-vf', filters.join(','));
+      }
+    }
+    args.push(options.playlistFilename);
+    return args;
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -129,23 +201,7 @@ export class BaseConfig implements VideoCodecSWConfig {
     const videoCodec = [TranscodeTarget.All, TranscodeTarget.Video].includes(target) ? this.getVideoCodec() : 'copy';
     const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioEncoder() : 'copy';
 
-    const options = [
-      '-c:v',
-      videoCodec,
-      '-c:a',
-      audioCodec,
-      // Makes a second pass moving the moov atom to the
-      // beginning of the file for improved playback speed.
-      '-movflags',
-      'faststart',
-      '-fps_mode',
-      'passthrough',
-      '-map',
-      `0:${videoStream.index}`,
-      '-map_metadata',
-      '-1',
-    ];
-
+    const options = ['-c:v', videoCodec, '-c:a', audioCodec, '-map', `0:${videoStream.index}`, '-map_metadata', '-1'];
     if (audioStream) {
       options.push('-map', `0:${audioStream.index}`);
     }
@@ -157,18 +213,22 @@ export class BaseConfig implements VideoCodecSWConfig {
     }
     if (this.getGopSize() > 0) {
       options.push('-g', `${this.getGopSize()}`);
+      if (this.tune.strictGop) {
+        options.push('-keyint_min', `${this.getGopSize()}`);
+      }
     }
-
-    if (
-      this.config.targetVideoCodec === VideoCodec.Hevc &&
-      (videoCodec !== 'copy' || videoStream.codecName === 'hevc')
-    ) {
+    const isHvc = (videoCodec === 'copy' ? videoStream.codecName : videoCodec) === VideoCodec.Hevc;
+    if (isHvc) {
       options.push('-tag:v', 'hvc1');
     }
 
     return options;
   }
 
+  getEncoderOptions(): string[] {
+    return [];
+  }
+
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
     if (this.shouldScale(videoStream)) {
@@ -272,25 +332,7 @@ export class BaseConfig implements VideoCodecSWConfig {
 
   getScaling(videoStream: VideoStreamInfo, mult = 2) {
     const targetResolution = this.getTargetResolution(videoStream);
-    return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
-  }
-
-  getSize(videoStream: VideoStreamInfo) {
-    const smaller = this.getTargetResolution(videoStream);
-    const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
-    let larger = Math.round(smaller * factor);
-    if (larger % 2 !== 0) {
-      larger -= 1;
-    }
-    return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller };
-  }
-
-  isVideoRotated(videoStream: VideoStreamInfo) {
-    return Math.abs(videoStream.rotation) === 90;
-  }
-
-  isVideoVertical(videoStream: VideoStreamInfo) {
-    return videoStream.height > videoStream.width || this.isVideoRotated(videoStream);
+    return isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
   }
 
   isBitrateConstrained() {
@@ -353,23 +395,18 @@ export class BaseConfig implements VideoCodecSWConfig {
   }
 }
 
-export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
+export class BaseHWConfig extends BaseConfig {
   protected device: string;
-  protected interfaces: VideoInterfaces;
 
   constructor(
     protected config: SystemConfigFFmpegDto,
-    interfaces: VideoInterfaces,
+    protected interfaces: VideoInterfaces,
+    tune?: VideoTuning,
   ) {
-    super(config);
-    this.interfaces = interfaces;
+    super(config, tune);
     this.device = this.getDevice(interfaces);
   }
 
-  getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.Hevc];
-  }
-
   validateDevices(devices: string[]) {
     if (devices.length === 0) {
       throw new Error('No /dev/dri devices found. If using Docker, make sure at least one /dev/dri device is mounted');
@@ -474,24 +511,32 @@ export class ThumbnailConfig extends BaseConfig {
 }
 
 export class H264Config extends BaseConfig {
-  getOutputThreadOptions() {
-    const options = super.getOutputThreadOptions();
-    if (this.config.threads === 1) {
-      options.push('-x264-params', 'frame-threads=1:pools=none');
+  getEncoderOptions(): string[] {
+    const out = this.getOutputThreadOptions();
+    if (this.tune.strictGop) {
+      out.push('-sc_threshold:v', '0');
     }
-
-    return options;
+    if (this.config.threads === 1) {
+      out.push('-x264-params', 'frame-threads=1:pools=none');
+    }
+    return out;
   }
 }
 
 export class HEVCConfig extends BaseConfig {
-  getOutputThreadOptions() {
-    const options = super.getOutputThreadOptions();
-    if (this.config.threads === 1) {
-      options.push('-x265-params', 'frame-threads=1:pools=none');
+  getEncoderOptions(): string[] {
+    const out: string[] = this.getOutputThreadOptions();
+    const params: string[] = [];
+    if (this.tune.strictGop) {
+      params.push('no-scenecut=1', 'no-open-gop=1');
     }
-
-    return options;
+    if (this.config.threads === 1) {
+      params.push('frame-threads=1', 'pools=none');
+    }
+    if (params.length > 0) {
+      out.push('-x265-params', params.join(':'));
+    }
+    return out;
   }
 }
 
@@ -520,8 +565,8 @@ export class VP9Config extends BaseConfig {
     return [`-${this.useCQP() ? 'q:v' : 'crf'}`, `${this.config.crf}`, '-b:v', `${bitrates.max}${bitrates.unit}`];
   }
 
-  getOutputThreadOptions() {
-    return ['-row-mt', '1', ...super.getOutputThreadOptions()];
+  getEncoderOptions(): string[] {
+    return ['-row-mt', '1', ...this.getOutputThreadOptions()];
   }
 
   eligibleForTwoPass() {
@@ -543,23 +588,22 @@ export class AV1Config extends BaseConfig {
   }
 
   getBitrateOptions() {
-    const options = ['-crf', `${this.config.crf}`];
-    const bitrates = this.getBitrateDistribution();
-    const svtparams = [];
-    if (this.config.threads > 0) {
-      svtparams.push(`lp=${this.config.threads}`);
-    }
-    if (bitrates.max > 0) {
-      svtparams.push(`mbr=${bitrates.max}${bitrates.unit}`);
-    }
-    if (svtparams.length > 0) {
-      options.push('-svtav1-params', svtparams.join(':'));
-    }
-    return options;
+    return ['-crf', `${this.config.crf}`];
   }
 
-  getOutputThreadOptions() {
-    return []; // Already set above with svtav1-params
+  getEncoderOptions(): string[] {
+    const params: string[] = [];
+    if (this.tune.lowLatency) {
+      params.push('hierarchical-levels=3', 'lookahead=0', 'enable-tf=0');
+    }
+    if (this.config.threads > 0) {
+      params.push(`lp=${this.config.threads}`);
+    }
+    const bitrates = this.getBitrateDistribution();
+    if (bitrates.max > 0) {
+      params.push(`mbr=${bitrates.max}${bitrates.unit}`);
+    }
+    return params.length > 0 ? ['-svtav1-params', params.join(':')] : [];
   }
 
   eligibleForTwoPass() {
@@ -572,10 +616,6 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
     return '0';
   }
 
-  getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Av1];
-  }
-
   getBaseInputOptions() {
     return ['-init_hw_device', `cuda=cuda:${this.device}`, '-filter_hw_device', 'cuda'];
   }
@@ -652,6 +692,14 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
     return [];
   }
 
+  getEncoderOptions(): string[] {
+    const out = this.getOutputThreadOptions();
+    if (this.tune.strictGop) {
+      out.push('-forced-idr', '1');
+    }
+    return out;
+  }
+
   getRefs() {
     const bframes = this.getBFrames();
     if (bframes > 0 && bframes < 3 && this.config.refs < 3) {
@@ -703,8 +751,8 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
     return ['-threads', '1'];
   }
 
-  getOutputThreadOptions() {
-    return [];
+  getEncoderOptions(): string[] {
+    return this.tune.strictGop ? ['-forced-idr', '1'] : [];
   }
 }
 
@@ -749,10 +797,6 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
-  getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1];
-  }
-
   // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
   getBFrames() {
     if (this.config.bframes < 0) {
@@ -775,6 +819,14 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
   getScaling(videoStream: VideoStreamInfo): string {
     return super.getScaling(videoStream, 1);
   }
+
+  getEncoderOptions(): string[] {
+    const out = this.getOutputThreadOptions();
+    if (this.tune.strictGop) {
+      out.push('-idr_interval', '0');
+    }
+    return out;
+  }
 }
 
 export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
@@ -888,13 +940,17 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
-  getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1];
-  }
-
   useCQP() {
     return this.config.cqMode !== CQMode.Icq || this.config.targetVideoCodec === VideoCodec.Vp9;
   }
+
+  getEncoderOptions(): string[] {
+    const out = this.getOutputThreadOptions();
+    if (this.tune.strictGop) {
+      out.push('-idr_interval', '0');
+    }
+    return out;
+  }
 }
 
 export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
@@ -988,10 +1044,6 @@ export class RkmppSwDecodeConfig extends BaseHWConfig {
     return ['-rc_mode', 'CQP', '-qp_init', `${this.config.crf}`];
   }
 
-  getSupportedCodecs() {
-    return [VideoCodec.H264, VideoCodec.Hevc];
-  }
-
   getVideoCodec(): string {
     return `${this.config.targetVideoCodec}_rkmpp`;
   }