feat(server): track video metadata (#28023)

* track video metadata

* earlier duration check

* revert colorspace change

* duplicate constant

* formatting

* linting

* add comments

* redundant variable

* simplify tests

* use totalDuration instead of format.duration

* medium tests

* install ffmpeg

* install noble

* update test-assets commit

* make timeBase non-nullable

* linting

* use proper smallint

* add ffmpeg to mise

* simplify duration

* regenerate migration

server/src/schema/index.ts

@@ -33,6 +33,7 @@ import { AlbumUserTable } from 'src/schema/tables/album-user.table';
 import { AlbumTable } from 'src/schema/tables/album.table';
 import { ApiKeyTable } from 'src/schema/tables/api-key.table';
 import { AssetAuditTable } from 'src/schema/tables/asset-audit.table';
+import { AssetAudioTable, AssetKeyframeTable, AssetVideoTable } from 'src/schema/tables/asset-av.table';
 import { AssetEditAuditTable } from 'src/schema/tables/asset-edit-audit.table';
 import { AssetEditTable } from 'src/schema/tables/asset-edit.table';
 import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
@@ -204,6 +205,9 @@ export interface DB {
   asset_metadata_audit: AssetMetadataAuditTable;
   asset_job_status: AssetJobStatusTable;
   asset_ocr: AssetOcrTable;
+  asset_audio: AssetAudioTable;
+  asset_video: AssetVideoTable;
+  asset_keyframe: AssetKeyframeTable;
   ocr_search: OcrSearchTable;
 
   face_search: FaceSearchTable;

server/src/schema/migrations/1777654048096-CreateAudioVideoTables.ts

@@ -0,0 +1,51 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`CREATE TABLE "asset_audio" (
+  "assetId" uuid NOT NULL,
+  "bitrate" integer NOT NULL,
+  "index" smallint NOT NULL,
+  "profile" smallint,
+  "codecName" text NOT NULL,
+  CONSTRAINT "asset_audio_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+  CONSTRAINT "asset_audio_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+  await sql`CREATE TABLE "asset_video" (
+  "assetId" uuid NOT NULL,
+  "bitrate" integer NOT NULL,
+  "frameCount" integer NOT NULL,
+  "timeBase" integer NOT NULL,
+  "index" smallint NOT NULL,
+  "profile" smallint,
+  "level" smallint,
+  "colorPrimaries" smallint NOT NULL,
+  "colorTransfer" smallint NOT NULL,
+  "colorMatrix" smallint NOT NULL,
+  "dvProfile" smallint,
+  "dvLevel" smallint,
+  "dvBlSignalCompatibilityId" smallint,
+  "codecName" text NOT NULL,
+  "formatName" text NOT NULL,
+  "formatLongName" text NOT NULL,
+  "pixelFormat" text NOT NULL,
+  CONSTRAINT "asset_video_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+  CONSTRAINT "asset_video_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+  await sql`CREATE TABLE "asset_keyframe" (
+  "assetId" uuid NOT NULL,
+  "pts" integer[] NOT NULL,
+  "accDuration" integer[] NOT NULL,
+  "ownDuration" integer[] NOT NULL,
+  "totalDuration" integer NOT NULL,
+  "packetCount" integer NOT NULL,
+  "outputFrames" integer NOT NULL,
+  CONSTRAINT "asset_keyframe_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+  CONSTRAINT "asset_keyframe_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`DROP TABLE "asset_audio";`.execute(db);
+  await sql`DROP TABLE "asset_video";`.execute(db);
+  await sql`DROP TABLE "asset_keyframe";`.execute(db);
+}

server/src/schema/tables/asset-av.table.ts

@@ -0,0 +1,98 @@
+import { Column, ForeignKeyColumn, Table } from '@immich/sql-tools';
+import { AssetTable } from 'src/schema/tables/asset.table';
+
+@Table('asset_audio')
+export class AssetAudioTable {
+  @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+  assetId!: string;
+
+  @Column({ type: 'integer' })
+  bitrate!: number;
+
+  @Column({ type: 'smallint' })
+  index!: number;
+
+  @Column({ type: 'smallint', nullable: true })
+  profile!: number | null;
+
+  @Column({ type: 'text' })
+  codecName!: string;
+}
+
+@Table('asset_video')
+export class AssetVideoTable {
+  @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+  assetId!: string;
+
+  @Column({ type: 'integer' })
+  bitrate!: number;
+
+  @Column({ type: 'integer' })
+  frameCount!: number;
+
+  @Column({ type: 'integer' })
+  timeBase!: number;
+
+  @Column({ type: 'smallint' })
+  index!: number;
+
+  @Column({ type: 'smallint', nullable: true })
+  profile!: number | null;
+
+  @Column({ type: 'smallint', nullable: true })
+  level!: number | null;
+
+  @Column({ type: 'smallint' })
+  colorPrimaries!: number;
+
+  @Column({ type: 'smallint' })
+  colorTransfer!: number;
+
+  @Column({ type: 'smallint' })
+  colorMatrix!: number;
+
+  @Column({ type: 'smallint', nullable: true })
+  dvProfile!: number | null;
+
+  @Column({ type: 'smallint', nullable: true })
+  dvLevel!: number | null;
+
+  @Column({ type: 'smallint', nullable: true })
+  dvBlSignalCompatibilityId!: number | null;
+
+  @Column({ type: 'text' })
+  codecName!: string;
+
+  @Column({ type: 'text' })
+  formatName!: string;
+
+  @Column({ type: 'text' })
+  formatLongName!: string;
+
+  @Column({ type: 'text' })
+  pixelFormat!: string;
+}
+
+@Table('asset_keyframe')
+export class AssetKeyframeTable {
+  @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+  assetId!: string;
+
+  @Column({ type: 'integer', array: true })
+  pts!: number[];
+
+  @Column({ type: 'integer', array: true })
+  accDuration!: number[];
+
+  @Column({ type: 'integer', array: true })
+  ownDuration!: number[];
+
+  @Column({ type: 'integer' })
+  totalDuration!: number;
+
+  @Column({ type: 'integer' })
+  packetCount!: number;
+
+  @Column({ type: 'integer' })
+  outputFrames!: number;
+}