feat(server): track video metadata (#28023)

* track video metadata

* earlier duration check

* revert colorspace change

* duplicate constant

* formatting

* linting

* add comments

* redundant variable

* simplify tests

* use totalDuration instead of format.duration

* medium tests

* install ffmpeg

* install noble

* update test-assets commit

* make timeBase non-nullable

* linting

* use proper smallint

* add ffmpeg to mise

* simplify duration

* regenerate migration
This commit is contained in:
Mert
2026-05-01 13:03:49 -04:00
committed by GitHub
parent c0898b96ca
commit f1d8ab8aae
31 changed files with 1697 additions and 389 deletions
+155 -29
View File
@@ -1,3 +1,5 @@
import { NotNull } from 'kysely';
import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum';
import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types';
const probeStubDefaultFormat: VideoFormat = {
@@ -15,9 +17,12 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
codecName: 'hevc',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
];
@@ -29,23 +34,13 @@ const probeStubDefault: VideoInfo = {
audioStreams: probeStubDefaultAudioStream,
};
export const probeStub = {
/** Fixtures in the shape `mediaRepository.probe()` returns (arrays of streams, raw ffprobe format). */
export const videoInfoStub = {
noVideoStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, videoStreams: [] }),
noAudioStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, audioStreams: [] }),
multipleVideoStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [
{
index: 0,
height: 1080,
width: 400,
codecName: 'hevc',
frameCount: 1,
rotation: 0,
isHDR: false,
bitrate: 100,
pixelFormat: 'yuv420p',
},
{
index: 1,
height: 1080,
@@ -53,9 +48,26 @@ export const probeStub = {
codecName: 'hevc',
frameCount: 2,
rotation: 0,
isHDR: false,
bitrate: 101,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
{
index: 0,
height: 1080,
width: 400,
codecName: 'hevc',
frameCount: 1,
rotation: 0,
bitrate: 100,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
{
index: 2,
@@ -64,18 +76,21 @@ export const probeStub = {
codecName: 'h7000',
frameCount: 3,
rotation: 0,
isHDR: false,
bitrate: 99,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
multipleAudioStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
{ index: 0, codecName: 'mp3', bitrate: 100 },
{ index: 1, codecName: 'mp3', bitrate: 101 },
{ index: 2, codecName: 'mp3', bitrate: 102 },
{ index: 1, codecName: 'mp3', bitrate: 101 },
{ index: 0, codecName: 'mp3', bitrate: 100 },
],
}),
noHeight: Object.freeze<VideoInfo>({
@@ -88,9 +103,12 @@ export const probeStub = {
codecName: 'hevc',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
@@ -104,9 +122,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
@@ -117,8 +138,10 @@ export const probeStub = {
videoStreamMTS: Object.freeze<VideoInfo>({
...probeStubDefault,
format: {
...probeStubDefaultFormat,
formatName: 'mpegts',
formatLongName: 'MPEG-TS (MPEG-2 Transport Stream)',
duration: 0,
bitrate: 0,
},
}),
videoStreamHDR: Object.freeze<VideoInfo>({
@@ -131,9 +154,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: true,
colorPrimaries: ColorPrimaries.Bt2020,
colorMatrix: ColorMatrix.Bt2020Nc,
colorTransfer: ColorTransfer.Smpte2084,
bitrate: 0,
pixelFormat: 'yuv420p10le',
timeBase: 600,
},
],
}),
@@ -147,9 +173,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
timeBase: 600,
},
],
}),
@@ -163,9 +192,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
timeBase: 600,
},
],
}),
@@ -179,9 +211,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 90,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
@@ -195,9 +230,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
@@ -211,9 +249,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
timeBase: 600,
},
],
}),
@@ -274,10 +315,95 @@ export const probeStub = {
videoStreams: [
{
...probeStubDefaultVideoStream[0],
colorPrimaries: 'reserved',
colorSpace: 'reserved',
colorTransfer: 'reserved',
colorPrimaries: ColorPrimaries.Reserved,
colorMatrix: ColorMatrix.Reserved,
colorTransfer: ColorTransfer.Reserved,
},
],
}),
videoStreamHDR10: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [
{
index: 0,
height: 2160,
width: 3840,
codecName: 'hevc',
profile: 2,
level: 153,
frameCount: 1208,
frameRate: 59.94,
rotation: 0,
bitrate: 64_000_000,
pixelFormat: 'yuv420p10le',
colorPrimaries: ColorPrimaries.Bt2020,
colorMatrix: ColorMatrix.Bt2020Nc,
colorTransfer: ColorTransfer.Smpte2084,
timeBase: 600,
},
],
}),
videoStreamDolbyVision: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [
{
index: 0,
height: 2160,
width: 3840,
codecName: 'hevc',
profile: 2,
level: 153,
frameCount: 1299,
frameRate: 59.94,
rotation: 0,
bitrate: 53_500_000,
pixelFormat: 'yuv420p10le',
colorPrimaries: ColorPrimaries.Bt2020,
colorMatrix: ColorMatrix.Bt2020Nc,
colorTransfer: ColorTransfer.AribStdB67,
dvProfile: DvProfile.Dvhe08,
dvLevel: 10,
dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
timeBase: 600,
},
],
}),
videoStreamWithProfileLevel: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [
{
...probeStubDefaultVideoStream[0],
codecName: 'h264',
profile: 100,
level: 40,
},
],
}),
audioStreamAAC: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
{
index: 1,
codecName: 'aac',
profile: 2,
bitrate: 128_000,
},
],
}),
};
interface SelectedStreams {
videoStream: VideoStreamInfo & { timeBase: NotNull };
audioStream: AudioStreamInfo | null;
format: VideoFormat;
}
const toSelectedStreams = (info: VideoInfo) => ({
videoStream: info.videoStreams[0] ?? null,
audioStream: info.audioStreams[0] ?? null,
format: info.format,
});
export const probeStub = Object.fromEntries(
Object.entries(videoInfoStub).map(([key, info]) => [key, toSelectedStreams(info)]),
) as Record<keyof typeof videoInfoStub, SelectedStreams>;
+5 -1
View File
@@ -1,9 +1,10 @@
import { Selectable, ShallowDehydrateObject } from 'kysely';
import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { ActivityTable } from 'src/schema/tables/activity.table';
import { AssetTable } from 'src/schema/tables/asset.table';
import { PartnerTable } from 'src/schema/tables/partner.table';
import { AudioStreamInfo, VideoFormat, VideoStreamInfo } from 'src/types';
import { AlbumFactory } from 'test/factories/album.factory';
import { AssetFaceFactory } from 'test/factories/asset-face.factory';
import { AssetFactory } from 'test/factories/asset.factory';
@@ -155,6 +156,9 @@ export const getForGenerateThumbnail = (asset: ReturnType<AssetFactory['build']>
files: asset.files.map((file) => getDehydrated(file)),
exifInfo: getDehydrated(asset.exifInfo),
edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[],
videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null,
audioStream: null as AudioStreamInfo | null,
format: null as VideoFormat | null,
});
export const getForAssetFace = (face: ReturnType<AssetFaceFactory['build']>) => ({
+11 -2
View File
@@ -35,6 +35,7 @@ import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { MapRepository } from 'src/repositories/map.repository';
import { MediaRepository } from 'src/repositories/media.repository';
import { MemoryRepository } from 'src/repositories/memory.repository';
import { MetadataRepository } from 'src/repositories/metadata.repository';
import { NotificationRepository } from 'src/repositories/notification.repository';
@@ -218,7 +219,7 @@ export class MediumTestContext<S extends BaseService = BaseService> {
}
async newExif(dto: Insertable<AssetExifTable>) {
const result = await this.get(AssetRepository).upsertExif(dto, { lockedPropertiesBehavior: 'override' });
const result = await this.get(AssetRepository).upsertExif({ exif: dto, lockedPropertiesBehavior: 'override' });
return { result };
}
@@ -362,7 +363,14 @@ export class ExifTestContext extends MediumTestContext<MetadataService> {
constructor(database: Kysely<DB>) {
super(MetadataService, {
database,
real: [AssetRepository, AssetJobRepository, MetadataRepository, SystemMetadataRepository, TagRepository],
real: [
AssetRepository,
AssetJobRepository,
MediaRepository,
MetadataRepository,
SystemMetadataRepository,
TagRepository,
],
mock: [ConfigRepository, EventRepository, LoggingRepository, MapRepository, StorageRepository],
});
@@ -445,6 +453,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
return new key(LoggingRepository.create());
}
case MediaRepository:
case MetadataRepository: {
return new key(LoggingRepository.create());
}
@@ -0,0 +1,142 @@
import { Kysely } from 'kysely';
import { resolve } from 'node:path';
import {
AacProfile,
AssetType,
ColorMatrix,
ColorPrimaries,
ColorTransfer,
DvProfile,
DvSignalCompatibility,
H264Profile,
HevcProfile,
} from 'src/enum';
import { DB } from 'src/schema';
import { ExifTestContext, testAssetsDir } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
let database: Kysely<DB>;
beforeAll(async () => {
database = await getKyselyDB();
});
const fixtures = [
{
file: 'eiffel-tower.mp4',
video: {
codecName: 'h264',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuv420p',
bitrate: 5_128_622,
frameCount: 557,
timeBase: 90_000,
index: 0,
profile: H264Profile.High,
level: 40,
colorPrimaries: ColorPrimaries.Smpte170M,
colorTransfer: ColorTransfer.Smpte170M,
colorMatrix: ColorMatrix.Smpte170M,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audio: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
keyframes: {
totalDuration: 2_012_441,
packetCount: 557,
outputFrames: 557,
pts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
accDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
ownDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
},
},
{
file: 'waterfall.mp4',
video: {
codecName: 'hevc',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuvj420p',
bitrate: 43_363_499,
frameCount: 309,
timeBase: 90_000,
index: 2,
profile: HevcProfile.Main,
level: 156,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audio: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
keyframes: {
totalDuration: 932_286,
packetCount: 309,
outputFrames: 309,
pts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
accDuration: [2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296],
ownDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
},
},
{
file: 'train.mov',
video: {
codecName: 'hevc',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuv420p10le',
bitrate: 12_595_191,
frameCount: 1229,
timeBase: 600,
index: 0,
profile: HevcProfile.Main10,
level: 123,
colorPrimaries: ColorPrimaries.Bt2020,
colorTransfer: ColorTransfer.AribStdB67,
colorMatrix: ColorMatrix.Bt2020Nc,
dvProfile: DvProfile.Dvhe08,
dvLevel: 5,
dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
},
audio: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
keyframes: {
totalDuration: 12_290,
packetCount: 1229,
outputFrames: 1303,
pts: [
0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210,
10_811, 11_411, 12_062, 12_703,
],
accDuration: [
10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180,
10_780, 11_380, 11_780, 12_100,
],
ownDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
},
},
];
const isExpected = <T extends keyof DB>(name: T, id: string, expected: Omit<DB[T], 'assetId'>) => {
const { table, ref } = database.dynamic;
const res = database.selectFrom(table(name).as('t')).selectAll().where(ref('assetId'), '=', id).executeTakeFirst();
return expect(res).resolves.toEqual({ ...expected, assetId: id });
};
describe('video metadata extraction', () => {
it.each(fixtures)('$file', async ({ file, video, audio, keyframes }) => {
const ctx = new ExifTestContext(database);
const { user } = await ctx.newUser();
const originalPath = resolve(testAssetsDir, 'videos', file);
const { asset } = await ctx.newAsset({ ownerId: user.id, originalPath, type: AssetType.Video });
await ctx.sut.handleMetadataExtraction({ id: asset.id });
await isExpected('asset_audio', asset.id, audio);
await isExpected('asset_video', asset.id, video);
await isExpected('asset_keyframe', asset.id, keyframes);
});
});
@@ -98,10 +98,10 @@ describe(AssetRepository.name, () => {
.executeTakeFirstOrThrow(),
).resolves.toEqual({ lockedProperties: ['dateTimeOriginal'] });
await sut.upsertExif(
{ assetId: asset.id, lockedProperties: ['description'] },
{ lockedPropertiesBehavior: 'append' },
);
await sut.upsertExif({
exif: { assetId: asset.id, lockedProperties: ['description'] },
lockedPropertiesBehavior: 'append',
});
await expect(
ctx.database
@@ -130,10 +130,10 @@ describe(AssetRepository.name, () => {
.executeTakeFirstOrThrow(),
).resolves.toEqual({ lockedProperties: ['dateTimeOriginal', 'description'] });
await sut.upsertExif(
{ assetId: asset.id, lockedProperties: ['description'] },
{ lockedPropertiesBehavior: 'append' },
);
await sut.upsertExif({
exif: { assetId: asset.id, lockedProperties: ['description'] },
lockedPropertiesBehavior: 'append',
});
await expect(
ctx.database
@@ -289,13 +289,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
await assetRepository.upsertExif(
updateLockedColumns({
await assetRepository.upsertExif({
exif: updateLockedColumns({
assetId: asset.id,
city: 'New City',
}),
{ lockedPropertiesBehavior: 'append' },
);
lockedPropertiesBehavior: 'append',
});
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
@@ -350,13 +350,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
await assetRepository.upsertExif(
updateLockedColumns({
await assetRepository.upsertExif({
exif: updateLockedColumns({
assetId: assetDelayedExif.id,
city: 'Delayed Exif',
}),
{ lockedPropertiesBehavior: 'append' },
);
lockedPropertiesBehavior: 'append',
});
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
@@ -11,6 +11,14 @@ export const newMediaRepositoryMock = (): Mocked<RepositoryInterface<MediaReposi
decodeImage: vitest.fn().mockResolvedValue({ data: Buffer.from(''), info: {} }),
extract: vitest.fn().mockResolvedValue(null),
probe: vitest.fn(),
probePackets: vitest.fn().mockResolvedValue({
totalDuration: 0,
packetCount: 0,
outputFrames: 0,
keyframePts: [],
keyframeAccDuration: [],
keyframeOwnDuration: [],
}),
transcode: vitest.fn(),
getImageMetadata: vitest.fn(),
};