fix(server): av typing (#28223)

* fix av typing, move fixtures to stub file

* fix tests
This commit is contained in:
Mert
2026-05-04 09:04:29 -04:00
committed by GitHub
parent 2015f95ff5
commit 01712cf0a7
8 changed files with 295 additions and 170 deletions
+220 -12
View File
@@ -1,5 +1,13 @@
import { NotNull } from 'kysely';
import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum';
import {
AacProfile,
ColorMatrix,
ColorPrimaries,
ColorTransfer,
DvProfile,
DvSignalCompatibility,
H264Profile,
HevcProfile,
} from 'src/enum';
import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types';
const probeStubDefaultFormat: VideoFormat = {
@@ -22,11 +30,17 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
];
const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 3, codecName: 'mp3', bitrate: 100 }];
const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 3, codecName: 'mp3', bitrate: 100, profile: null }];
const probeStubDefault: VideoInfo = {
format: probeStubDefaultFormat,
@@ -53,7 +67,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
{
index: 0,
@@ -67,7 +87,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
{
index: 2,
@@ -81,16 +107,22 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
multipleAudioStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
{ index: 2, codecName: 'mp3', bitrate: 102 },
{ index: 1, codecName: 'mp3', bitrate: 101 },
{ index: 0, codecName: 'mp3', bitrate: 100 },
{ index: 2, codecName: 'mp3', bitrate: 102, profile: null },
{ index: 1, codecName: 'mp3', bitrate: 101, profile: null },
{ index: 0, codecName: 'mp3', bitrate: 100, profile: null },
],
}),
noHeight: Object.freeze<VideoInfo>({
@@ -108,7 +140,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -127,7 +165,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: HevcProfile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -159,7 +203,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Smpte2084,
bitrate: 0,
pixelFormat: 'yuv420p10le',
frameRate: 60,
timeBase: 600,
profile: H264Profile.High10,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -178,7 +228,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
frameRate: 60,
timeBase: 600,
profile: H264Profile.High10,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -197,7 +253,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
frameRate: 60,
timeBase: 600,
profile: H264Profile.High10,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -216,7 +278,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: H264Profile.High,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -235,7 +303,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: H264Profile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -254,27 +328,33 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
frameRate: 60,
timeBase: 600,
profile: H264Profile.Main,
level: null,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
audioStreamAac: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100 }],
audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100, profile: AacProfile.Lc }],
}),
audioStreamMp3: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100 }],
audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100, profile: null }],
}),
audioStreamOpus: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100 }],
audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100, profile: null }],
}),
audioStreamUnknown: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
{ index: 0, codecName: 'aac', bitrate: 100 },
{ index: 1, codecName: 'unknown', bitrate: 200 },
{ index: 0, codecName: 'aac', bitrate: 100, profile: AacProfile.Lc },
{ index: 1, codecName: 'unknown', bitrate: 200, profile: null },
],
}),
matroskaContainer: Object.freeze<VideoInfo>({
@@ -340,6 +420,9 @@ export const videoInfoStub = {
colorMatrix: ColorMatrix.Bt2020Nc,
colorTransfer: ColorTransfer.Smpte2084,
timeBase: 600,
dvBlSignalCompatibilityId: null,
dvLevel: null,
dvProfile: null,
},
],
}),
@@ -393,7 +476,7 @@ export const videoInfoStub = {
};
interface SelectedStreams {
videoStream: VideoStreamInfo & { timeBase: NotNull };
videoStream: VideoStreamInfo & { timeBase: number };
audioStream: AudioStreamInfo | null;
format: VideoFormat;
}
@@ -407,3 +490,128 @@ const toSelectedStreams = (info: VideoInfo) => ({
export const probeStub = Object.fromEntries(
Object.entries(videoInfoStub).map(([key, info]) => [key, toSelectedStreams(info)]),
) as Record<keyof typeof videoInfoStub, SelectedStreams>;
export const eiffelTower = {
originalPath: 'eiffel-tower.mp4',
videoStream: {
index: 0,
width: 1080,
height: 1920,
rotation: 0,
codecName: 'h264',
profile: H264Profile.High,
level: 40,
frameCount: 557,
frameRate: 24.908_004_845_459_07,
timeBase: 90_000,
bitrate: 5_128_622,
pixelFormat: 'yuv420p',
colorPrimaries: ColorPrimaries.Smpte170M,
colorTransfer: ColorTransfer.Smpte170M,
colorMatrix: ColorMatrix.Smpte170M,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audioStream: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
packets: {
totalDuration: 2_012_441,
packetCount: 557,
outputFrames: 557,
keyframePts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
keyframeAccDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
keyframeOwnDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
},
format: {
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
duration: 22_616,
bitrate: 5_128_622,
},
};
export const waterfall = {
originalPath: 'waterfall.mp4',
videoStream: {
index: 2,
width: 3840,
height: 2160,
rotation: -90,
codecName: 'hevc',
profile: HevcProfile.Main,
level: 156,
frameCount: 309,
frameRate: 29.829_901_982_867_92,
timeBase: 90_000,
bitrate: 43_363_499,
pixelFormat: 'yuvj420p',
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audioStream: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
packets: {
totalDuration: 932_286,
packetCount: 309,
outputFrames: 309,
keyframePts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
keyframeAccDuration: [
2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296,
],
keyframeOwnDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
},
format: {
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
duration: 10_359,
bitrate: 43_363_499,
},
};
export const train = {
originalPath: 'train.mov',
videoStream: {
index: 0,
width: 1920,
height: 1080,
rotation: -90,
codecName: 'hevc',
profile: HevcProfile.Main10,
level: 123,
frameCount: 1229,
frameRate: 56.536_072_989_342_94,
timeBase: 600,
bitrate: 12_595_191,
pixelFormat: 'yuv420p10le',
colorPrimaries: ColorPrimaries.Bt2020,
colorTransfer: ColorTransfer.AribStdB67,
colorMatrix: ColorMatrix.Bt2020Nc,
dvProfile: DvProfile.Dvhe08,
dvLevel: 5,
dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
},
audioStream: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
packets: {
totalDuration: 12_290,
packetCount: 1229,
outputFrames: 1303,
keyframePts: [
0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210, 10_811,
11_411, 12_062, 12_703,
],
keyframeAccDuration: [
10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180, 10_780,
11_380, 11_780, 12_100,
],
keyframeOwnDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
},
format: {
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
duration: 21_738,
bitrate: 12_595_191,
},
};
+2 -2
View File
@@ -1,4 +1,4 @@
import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely';
import { Selectable, ShallowDehydrateObject } from 'kysely';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { ActivityTable } from 'src/schema/tables/activity.table';
@@ -156,7 +156,7 @@ export const getForGenerateThumbnail = (asset: ReturnType<AssetFactory['build']>
files: asset.files.map((file) => getDehydrated(file)),
exifInfo: getDehydrated(asset.exifInfo),
edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[],
videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null,
videoStream: null as (VideoStreamInfo & { timeBase: number }) | null,
audioStream: null as AudioStreamInfo | null,
format: null as VideoFormat | null,
});
+20 -120
View File
@@ -1,17 +1,9 @@
import { Kysely } from 'kysely';
import { resolve } from 'node:path';
import {
AacProfile,
AssetType,
ColorMatrix,
ColorPrimaries,
ColorTransfer,
DvProfile,
DvSignalCompatibility,
H264Profile,
HevcProfile,
} from 'src/enum';
import { AssetType } from 'src/enum';
import { DB } from 'src/schema';
import { withAudioStream, withVideoFormat, withVideoPackets, withVideoStream } from 'src/utils/database';
import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub';
import { ExifTestContext, testAssetsDir } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
@@ -21,122 +13,30 @@ beforeAll(async () => {
database = await getKyselyDB();
});
const fixtures = [
{
file: 'eiffel-tower.mp4',
video: {
codecName: 'h264',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuv420p',
bitrate: 5_128_622,
frameCount: 557,
timeBase: 90_000,
index: 0,
profile: H264Profile.High,
level: 40,
colorPrimaries: ColorPrimaries.Smpte170M,
colorTransfer: ColorTransfer.Smpte170M,
colorMatrix: ColorMatrix.Smpte170M,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audio: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
keyframes: {
totalDuration: 2_012_441,
packetCount: 557,
outputFrames: 557,
pts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
accDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
ownDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
},
},
{
file: 'waterfall.mp4',
video: {
codecName: 'hevc',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuvj420p',
bitrate: 43_363_499,
frameCount: 309,
timeBase: 90_000,
index: 2,
profile: HevcProfile.Main,
level: 156,
colorPrimaries: ColorPrimaries.Bt709,
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
dvProfile: null,
dvLevel: null,
dvBlSignalCompatibilityId: null,
},
audio: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
keyframes: {
totalDuration: 932_286,
packetCount: 309,
outputFrames: 309,
pts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
accDuration: [2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296],
ownDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
},
},
{
file: 'train.mov',
video: {
codecName: 'hevc',
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
pixelFormat: 'yuv420p10le',
bitrate: 12_595_191,
frameCount: 1229,
timeBase: 600,
index: 0,
profile: HevcProfile.Main10,
level: 123,
colorPrimaries: ColorPrimaries.Bt2020,
colorTransfer: ColorTransfer.AribStdB67,
colorMatrix: ColorMatrix.Bt2020Nc,
dvProfile: DvProfile.Dvhe08,
dvLevel: 5,
dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
},
audio: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
keyframes: {
totalDuration: 12_290,
packetCount: 1229,
outputFrames: 1303,
pts: [
0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210,
10_811, 11_411, 12_062, 12_703,
],
accDuration: [
10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180,
10_780, 11_380, 11_780, 12_100,
],
ownDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
},
},
];
const isExpected = <T extends keyof DB>(name: T, id: string, expected: Omit<DB[T], 'assetId'>) => {
const { table, ref } = database.dynamic;
const res = database.selectFrom(table(name).as('t')).selectAll().where(ref('assetId'), '=', id).executeTakeFirst();
return expect(res).resolves.toEqual({ ...expected, assetId: id });
};
const fixtures = [eiffelTower, waterfall, train];
describe('video metadata extraction', () => {
it.each(fixtures)('$file', async ({ file, video, audio, keyframes }) => {
it.each(fixtures)('$originalPath', async ({ originalPath: path, videoStream, audioStream, packets, format }) => {
const ctx = new ExifTestContext(database);
const { user } = await ctx.newUser();
const originalPath = resolve(testAssetsDir, 'videos', file);
const originalPath = resolve(testAssetsDir, 'videos', path);
const { asset } = await ctx.newAsset({ ownerId: user.id, originalPath, type: AssetType.Video });
await ctx.sut.handleMetadataExtraction({ id: asset.id });
await isExpected('asset_audio', asset.id, audio);
await isExpected('asset_video', asset.id, video);
await isExpected('asset_keyframe', asset.id, keyframes);
const result = await database
.selectFrom('asset')
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
.innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
.innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
.leftJoin('asset_audio', 'asset.id', 'asset_audio.assetId')
.where('asset.id', '=', asset.id)
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
.select((eb) => withAudioStream(eb).as('audioStream'))
.select((eb) => withVideoPackets(eb).$notNull().as('packets'))
.select((eb) => withVideoFormat(eb).$notNull().as('format'))
.executeTakeFirst();
expect(result).toEqual({ videoStream, audioStream, packets, format });
});
});