feat(server): add OpenTelemetry tracing and metrics support

Adds comprehensive OpenTelemetry instrumentation for better observability: Tracing: - HTTP requests via @opentelemetry/instrumentation-http - NestJS controllers via @opentelemetry/instrumentation-nestjs-core - Redis operations via @opentelemetry/instrumentation-ioredis - BullMQ job queues via bullmq-otel - Database queries via Kysely log callback with accurate timing - File operations (send, stream, read) - Media processing (thumbnails, thumbhash, EXIF operations) - Metadata extraction Metrics: - Database connection pool usage (used/idle connections) The SDK is initialized in telemetry-preload.ts which is preloaded via --require before the main app starts. This ensures http instrumentation hooks are in place before any http module is imported. Enable tracing by setting OTEL_EXPORTER_OTLP_ENDPOINT environment variable. Also adds TraceContext to IBaseJob for future distributed trace propagation across BullMQ job boundaries.
2026-05-18 03:10:24 +03:00 · 2026-01-27 16:54:23 +00:00
parent d6c5a382f8
commit e03ed9b387
21 changed files with 3270 additions and 249 deletions
@@ -6,4 +6,17 @@ if [[ "$IMMICH_ENV" == "production" ]]; then
 fi

 cd /usr/src/app || exit
+
+# Compile telemetry-preload so it can be required before the app starts.
+# This is necessary for OpenTelemetry http instrumentation to work correctly.
+mkdir -p dist
+pnpm --filter immich exec tsc src/telemetry-preload.ts \
+  --outDir dist \
+  --esModuleInterop \
+  --module node16 \
+  --moduleResolution node16 \
+  --target ES2022 \
+  --skipLibCheck
+
+export NODE_OPTIONS="${NODE_OPTIONS} --require ./dist/telemetry-preload.js"
 pnpm --filter immich exec nest start --debug "0.0.0.0:9230" --watch -- "$@"
@@ -39,7 +39,7 @@ else
 fi

 if [ -f "${SERVER_HOME}/dist/main.js" ]; then
-  exec node "${SERVER_HOME}/dist/main.js" "$@"
+  exec node --require "${SERVER_HOME}/dist/telemetry-preload.js" "${SERVER_HOME}/dist/main.js" "$@"
 else
  echo "Error: ${SERVER_HOME}/dist/main.js not found"
  if [ "$IMMICH_ENV" = "development" ]; then
@@ -45,14 +45,16 @@
    "@nestjs/websockets": "^11.0.4",
    "@opentelemetry/api": "^1.9.0",
    "@opentelemetry/context-async-hooks": "^2.0.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.211.0",
    "@opentelemetry/exporter-prometheus": "^0.210.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.211.0",
    "@opentelemetry/instrumentation-http": "^0.210.0",
    "@opentelemetry/instrumentation-ioredis": "^0.58.0",
    "@opentelemetry/instrumentation-nestjs-core": "^0.56.0",
-    "@opentelemetry/instrumentation-pg": "^0.62.0",
    "@opentelemetry/resources": "^2.0.1",
    "@opentelemetry/sdk-metrics": "^2.0.1",
    "@opentelemetry/sdk-node": "^0.210.0",
+    "@opentelemetry/sdk-trace-base": "^2.5.0",
    "@opentelemetry/semantic-conventions": "^1.34.0",
    "@react-email/components": "^0.5.0",
    "@react-email/render": "^1.1.2",
@@ -63,6 +65,7 @@
    "bcrypt": "^6.0.0",
    "body-parser": "^2.2.0",
    "bullmq": "^5.51.0",
+    "bullmq-otel": "^1.1.0",
    "chokidar": "^4.0.3",
    "class-transformer": "^0.5.1",
    "class-validator": "^0.14.0",
@@ -47,6 +47,13 @@ export enum AssetFileType {
  Sidecar = 'sidecar',
 }

+export enum GeneratedImageType {
+  Thumbnail = 'thumbnail',
+  Preview = 'preview',
+  Fullsize = 'fullsize',
+  Person = 'person',
+}
+
 export enum AlbumUserRole {
  Editor = 'editor',
  Viewer = 'viewer',
@@ -8,6 +8,7 @@ import { RedisOptions } from 'ioredis';
 import { CLS_ID, ClsModuleOptions } from 'nestjs-cls';
 import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
 import { join } from 'node:path';
+import { bullMQTelemetry } from 'src/telemetry-preload';
 import { citiesFile, excludePaths, IWorker } from 'src/constants';
 import { Telemetry } from 'src/decorators';
 import { EnvDto } from 'src/dtos/env.dto';
@@ -262,6 +263,7 @@ const getEnv = (): EnvData => {
          removeOnComplete: true,
          removeOnFail: false,
        },
+        telemetry: bullMQTelemetry,
      },
      queues: Object.values(QueueName).map((name) => ({ name })),
    },
@@ -1,9 +1,13 @@
 import { Injectable } from '@nestjs/common';
+import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
 import { Duration } from 'luxon';
 import { readFile } from 'node:fs/promises';
 import { MachineLearningConfig } from 'src/config';
 import { CLIPConfig } from 'src/dtos/model-config.dto';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import { Traced } from 'src/utils/instrumentation';
+
+const tracer = trace.getTracer('immich');

 export interface BoundingBox {
  x1: number;
@@ -162,36 +166,60 @@ export class MachineLearningRepository {
  }

  private async predict<T>(payload: ModelPayload, config: MachineLearningRequest): Promise<T> {
-    const formData = await this.getFormData(payload, config);
+    const taskType = Object.keys(config)[0] as ModelTask;

-    for (const url of [
-      // try healthy servers first
-      ...this.config.urls.filter((url) => this.isHealthy(url)),
-      ...this.config.urls.filter((url) => !this.isHealthy(url)),
-    ]) {
-      try {
-        const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData });
-        if (response.ok) {
-          this.setHealthy(url, true);
-          return response.json();
+    return tracer.startActiveSpan('ml.predict', { kind: SpanKind.CLIENT }, context.active(), async (span: Span) => {
+      span.setAttribute('ml.task', taskType);
+
+      const formData = await this.getFormData(payload, config);
+      let attemptCount = 0;
+      let successUrl: string | undefined;
+
+      const urls = [
+        ...this.config.urls.filter((url) => this.isHealthy(url)),
+        ...this.config.urls.filter((url) => !this.isHealthy(url)),
+      ];
+
+      for (const url of urls) {
+        attemptCount++;
+        try {
+          const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData });
+          if (response.ok) {
+            this.setHealthy(url, true);
+            successUrl = url;
+            span.setAttribute('ml.url', url);
+            span.setAttribute('ml.attempts', attemptCount);
+            span.setStatus({ code: SpanStatusCode.OK });
+            span.end();
+            return response.json();
+          }
+
+          this.logger.warn(
+            `Machine learning request to "${url}" failed with status ${response.status}: ${response.statusText}`,
+          );
+        } catch (error: Error | unknown) {
+          this.logger.warn(
+            `Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`,
+          );
        }

-        this.logger.warn(
-          `Machine learning request to "${url}" failed with status ${response.status}: ${response.statusText}`,
-        );
-      } catch (error: Error | unknown) {
-        this.logger.warn(
-          `Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`,
-        );
+        this.setHealthy(url, false);
      }

-      this.setHealthy(url, false);
-    }
-
-    throw new Error(`Machine learning request '${JSON.stringify(config)}' failed for all URLs`);
+      span.setAttribute('ml.attempts', attemptCount);
+      const errorMessage = `Machine learning request '${JSON.stringify(config)}' failed for all URLs`;
+      span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage });
+      span.end();
+      throw new Error(errorMessage);
+    });
  }

+  @Traced({ name: 'ml.detectFaces', kind: SpanKind.CLIENT })
  async detectFaces(imagePath: string, { modelName, minScore }: FaceDetectionOptions) {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('ml.modelName', modelName);
+    span?.setAttribute('ml.minScore', minScore);
+
    const request = {
      [ModelTask.FACIAL_RECOGNITION]: {
        [ModelType.DETECTION]: { modelName, options: { minScore } },
@@ -206,19 +234,36 @@ export class MachineLearningRepository {
    };
  }

+  @Traced({ name: 'ml.encodeImage', kind: SpanKind.CLIENT })
  async encodeImage(imagePath: string, { modelName }: CLIPConfig) {
+    trace.getActiveSpan()?.setAttribute('ml.modelName', modelName);
+
    const request = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: { modelName } } };
    const response = await this.predict<ClipVisualResponse>({ imagePath }, request);
    return response[ModelTask.SEARCH];
  }

+  @Traced({ name: 'ml.encodeText', kind: SpanKind.CLIENT })
  async encodeText(text: string, { language, modelName }: TextEncodingOptions) {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('ml.modelName', modelName);
+    if (language) {
+      span?.setAttribute('ml.language', language);
+    }
+
    const request = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: { modelName, options: { language } } } };
    const response = await this.predict<ClipTextualResponse>({ text }, request);
    return response[ModelTask.SEARCH];
  }

+  @Traced({ name: 'ml.ocr', kind: SpanKind.CLIENT })
  async ocr(imagePath: string, { modelName, minDetectionScore, minRecognitionScore, maxResolution }: OcrOptions) {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('ml.modelName', modelName);
+    span?.setAttribute('ml.minDetectionScore', minDetectionScore);
+    span?.setAttribute('ml.minRecognitionScore', minRecognitionScore);
+    span?.setAttribute('ml.maxResolution', maxResolution);
+
    const request = {
      [ModelTask.OCR]: {
        [ModelType.DETECTION]: { modelName, options: { minScore: minDetectionScore, maxResolution } },
@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
 import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored';
 import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
 import { Duration } from 'luxon';
@@ -19,6 +20,7 @@ import {
  TranscodeCommand,
  VideoInfo,
 } from 'src/types';
+import { Traced } from 'src/utils/instrumentation';
 import { handlePromiseError } from 'src/utils/misc';
 import { createAffineMatrix } from 'src/utils/transform';

@@ -26,9 +28,17 @@ const probe = (input: string, options: string[]): Promise<FfprobeData> =>
  new Promise((resolve, reject) =>
    ffmpeg.ffprobe(input, options, (error, data) => (error ? reject(error) : resolve(data))),
  );
+const tracer = trace.getTracer('immich');
 sharp.concurrency(0);
 sharp.cache({ files: 0 });

+const EXTRACTION_METHODS = [
+  { tag: 'JpgFromRaw2', format: RawExtractedFormat.Jpeg },
+  { tag: 'JpgFromRaw', format: RawExtractedFormat.Jpeg },
+  { tag: 'PreviewJXL', format: RawExtractedFormat.Jxl },
+  { tag: 'PreviewImage', format: RawExtractedFormat.Jpeg },
+] as const;
+
 type ProgressEvent = {
  frames: number;
  currentFps: number;
@@ -49,81 +59,66 @@ export class MediaRepository {
    this.logger.setContext(MediaRepository.name);
  }

-  /**
-   *
-   * @param input file path to the input image
-   * @returns ExtractResult if succeeded, or null if failed
-   */
+  @Traced('media.exiftool.extract')
  async extract(input: string): Promise<ExtractResult | null> {
-    try {
-      const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw2', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
-    } catch (error: any) {
-      this.logger.debug(`Could not extract JpgFromRaw2 buffer from image, trying JPEG from RAW next: ${error}`);
-    }
-
-    try {
-      const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
-    } catch (error: any) {
-      this.logger.debug(`Could not extract JPEG buffer from image, trying PreviewJXL next: ${error}`);
-    }
-
-    try {
-      const buffer = await exiftool.extractBinaryTagToBuffer('PreviewJXL', input);
-      return { buffer, format: RawExtractedFormat.Jxl };
-    } catch (error: any) {
-      this.logger.debug(`Could not extract PreviewJXL buffer from image, trying PreviewImage next: ${error}`);
-    }
-
-    try {
-      const buffer = await exiftool.extractBinaryTagToBuffer('PreviewImage', input);
-      return { buffer, format: RawExtractedFormat.Jpeg };
-    } catch (error: any) {
-      this.logger.debug(`Could not extract preview buffer from image: ${error}`);
-      return null;
+    for (const { tag, format } of EXTRACTION_METHODS) {
+      try {
+        const buffer = await exiftool.extractBinaryTagToBuffer(tag, input);
+        trace.getActiveSpan()?.setAttribute('exiftool.extractedTag', tag);
+        return { buffer, format };
+      } catch {
+        this.logger.debug(`Could not extract ${tag} from image, trying next method`);
+      }
    }
+    trace.getActiveSpan()?.setAttribute('exiftool.extractedTag', 'none');
+    return null;
  }

+  @Traced('media.exiftool.writeExif')
  async writeExif(tags: Partial<Exif>, output: string): Promise<boolean> {
-    try {
-      const tagsToWrite: WriteTags = {
-        ExifImageWidth: tags.exifImageWidth,
-        ExifImageHeight: tags.exifImageHeight,
-        DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()),
-        ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()),
-        TimeZone: tags.timeZone,
-        GPSLatitude: tags.latitude,
-        GPSLongitude: tags.longitude,
-        ProjectionType: tags.projectionType,
-        City: tags.city,
-        Country: tags.country,
-        Make: tags.make,
-        Model: tags.model,
-        LensModel: tags.lensModel,
-        Fnumber: tags.fNumber?.toFixed(1),
-        FocalLength: tags.focalLength?.toFixed(1),
-        ISO: tags.iso,
-        ExposureTime: tags.exposureTime,
-        ProfileDescription: tags.profileDescription,
-        ColorSpace: tags.colorspace,
-        Rating: tags.rating,
-        // specially convert Orientation to numeric Orientation# for exiftool
-        'Orientation#': tags.orientation ? Number(tags.orientation) : undefined,
-      };
+    const tagsToWrite: WriteTags = {
+      ExifImageWidth: tags.exifImageWidth,
+      ExifImageHeight: tags.exifImageHeight,
+      DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()),
+      ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()),
+      TimeZone: tags.timeZone,
+      GPSLatitude: tags.latitude,
+      GPSLongitude: tags.longitude,
+      ProjectionType: tags.projectionType,
+      City: tags.city,
+      Country: tags.country,
+      Make: tags.make,
+      Model: tags.model,
+      LensModel: tags.lensModel,
+      Fnumber: tags.fNumber?.toFixed(1),
+      FocalLength: tags.focalLength?.toFixed(1),
+      ISO: tags.iso,
+      ExposureTime: tags.exposureTime,
+      ProfileDescription: tags.profileDescription,
+      ColorSpace: tags.colorspace,
+      Rating: tags.rating,
+      'Orientation#': tags.orientation ? Number(tags.orientation) : undefined,
+    };

+    try {
      await exiftool.write(output, tagsToWrite, {
        ignoreMinorErrors: true,
        writeArgs: ['-overwrite_original'],
      });
      return true;
-    } catch (error: any) {
-      this.logger.warn(`Could not write exif data to image: ${error.message}`);
+    } catch (error: unknown) {
+      const span = trace.getActiveSpan();
+      span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      span?.recordException(error as Error);
+      this.logger.warn(`Could not write exif data to image: ${error}`);
      return false;
    }
  }

+  @Traced('media.exiftool.copyTagGroup')
  async copyTagGroup(tagGroup: string, source: string, target: string): Promise<boolean> {
+    trace.getActiveSpan()?.setAttribute('exiftool.tagGroup', tagGroup);
+
    try {
      await exiftool.write(
        target,
@@ -134,15 +129,28 @@ export class MediaRepository {
        },
      );
      return true;
-    } catch (error: any) {
-      this.logger.warn(`Could not copy tag data to image: ${error.message}`);
+    } catch (error: unknown) {
+      const span = trace.getActiveSpan();
+      span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      span?.recordException(error as Error);
+      this.logger.warn(`Could not copy tag data to image: ${error}`);
      return false;
    }
  }

+  @Traced('media.decodeImage')
  async decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('media.colorspace', options.colorspace);
+    if (options.size !== undefined) {
+      span?.setAttribute('media.size', options.size);
+    }
+
    const pipeline = await this.getImageDecodingPipeline(input, options);
-    return pipeline.raw().toBuffer({ resolveWithObject: true });
+    const result = await pipeline.raw().toBuffer({ resolveWithObject: true });
+    span?.setAttribute('media.output.width', result.info.width);
+    span?.setAttribute('media.output.height', result.info.height);
+    return result;
  }

  private async applyEdits(pipeline: sharp.Sharp, edits: AssetEditActionItem[]): Promise<sharp.Sharp> {
@@ -170,11 +178,21 @@ export class MediaRepository {
    return pipeline;
  }

+  @Traced('media.generateThumbnail')
  async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('media.imageType', options.imageType ?? 'unknown');
+    span?.setAttribute('media.format', options.format);
+    span?.setAttribute('media.quality', options.quality);
+    span?.setAttribute('media.size', options.size ?? 0);
+    span?.setAttribute('media.colorspace', options.colorspace);
+    if (options.progressive !== undefined) {
+      span?.setAttribute('media.progressive', options.progressive);
+    }
+
    const pipeline = await this.getImageDecodingPipeline(input, options);
    const decoded = pipeline.toFormat(options.format, {
      quality: options.quality,
-      // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
      chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0',
      progressive: options.progressive,
    });
@@ -215,7 +233,11 @@ export class MediaRepository {
    return pipeline;
  }

+  @Traced('media.generateThumbhash')
  async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('media.colorspace', options.colorspace);
+
    const [{ rgbaToThumbHash }, decodingPipeline] = await Promise.all([
      import('thumbhash'),
      this.getImageDecodingPipeline(input, {
@@ -229,12 +251,23 @@ export class MediaRepository {
    const pipeline = decodingPipeline.resize(100, 100, { fit: 'inside', withoutEnlargement: true }).raw().ensureAlpha();

    const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
+    span?.setAttribute('media.thumbhash.width', info.width);
+    span?.setAttribute('media.thumbhash.height', info.height);

    return Buffer.from(rgbaToThumbHash(info.width, info.height, data));
  }

+  @Traced('ffmpeg.probe')
  async probe(input: string, options?: ProbeOptions): Promise<VideoInfo> {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('ffmpeg.countFrames', options?.countFrames ?? false);
+
    const results = await probe(input, options?.countFrames ? ['-count_packets'] : []); // gets frame count quickly: https://stackoverflow.com/a/28376817
+
+    span?.setAttribute('ffmpeg.duration', this.parseFloat(results.format.duration));
+    span?.setAttribute('ffmpeg.videoStreams', results.streams.filter((s) => s.codec_type === 'video').length);
+    span?.setAttribute('ffmpeg.audioStreams', results.streams.filter((s) => s.codec_type === 'audio').length);
+
    return {
      format: {
        formatName: results.format.format_name,
@@ -272,17 +305,39 @@ export class MediaRepository {
  }

  transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise<void> {
+    const span = tracer.startSpan('ffmpeg.transcode', { kind: SpanKind.INTERNAL }, context.active());
+    span.setAttribute('ffmpeg.twoPass', options.twoPass ?? false);
+    span.setAttribute('ffmpeg.frameCount', options.progress.frameCount);
+
+    const endSpan = (error?: Error) => {
+      if (error) {
+        span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+        span.recordException(error);
+      } else {
+        span.setStatus({ code: SpanStatusCode.OK });
+      }
+      span.end();
+    };
+
    if (!options.twoPass) {
      return new Promise((resolve, reject) => {
        this.configureFfmpegCall(input, output, options)
-          .on('error', reject)
-          .on('end', () => resolve())
+          .on('error', (error) => {
+            endSpan(error);
+            reject(error);
+          })
+          .on('end', () => {
+            endSpan();
+            resolve();
+          })
          .run();
      });
    }

    if (typeof output !== 'string') {
-      throw new TypeError('Two-pass transcoding does not support writing to a stream');
+      const error = new TypeError('Two-pass transcoding does not support writing to a stream');
+      endSpan(error);
+      throw error;
    }

    // two-pass allows for precise control of bitrate at the cost of running twice
@@ -293,24 +348,37 @@ export class MediaRepository {
        .addOptions('-pass', '1')
        .addOptions('-passlogfile', output)
        .addOptions('-f null')
-        .on('error', reject)
+        .on('error', (error) => {
+          endSpan(error);
+          reject(error);
+        })
        .on('end', () => {
          // second pass
          this.configureFfmpegCall(input, output, options)
            .addOptions('-pass', '2')
            .addOptions('-passlogfile', output)
-            .on('error', reject)
+            .on('error', (error) => {
+              endSpan(error);
+              reject(error);
+            })
            .on('end', () => handlePromiseError(fs.unlink(`${output}-0.log`), this.logger))
            .on('end', () => handlePromiseError(fs.rm(`${output}-0.log.mbtree`, { force: true }), this.logger))
-            .on('end', () => resolve())
+            .on('end', () => {
+              endSpan();
+              resolve();
+            })
            .run();
        })
        .run();
    });
  }

+  @Traced('media.getImageDimensions')
  async getImageDimensions(input: string | Buffer): Promise<ImageDimensions> {
    const { width = 0, height = 0 } = await sharp(input).metadata();
+    const span = trace.getActiveSpan();
+    span?.setAttribute('media.width', width);
+    span?.setAttribute('media.height', height);
    return { width, height };
  }

@@ -1,7 +1,9 @@
 import { Injectable } from '@nestjs/common';
+import { SpanStatusCode, trace } from '@opentelemetry/api';
 import { BinaryField, DefaultReadTaskOptions, ExifTool, Tags } from 'exiftool-vendored';
 import geotz from 'geo-tz';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import { Traced } from 'src/utils/instrumentation';
 import { mimeTypes } from 'src/utils/mime-types';

 interface ExifDuration {
@@ -104,22 +106,34 @@ export class MetadataRepository {
    await this.exiftool.end();
  }

-  readTags(path: string): Promise<ImmichTags> {
-    const args = mimeTypes.isVideo(path) ? ['-ee'] : [];
-    return this.exiftool.read(path, args).catch((error) => {
-      this.logger.warn(`Error reading exif data (${path}): ${error}\n${error?.stack}`);
+  @Traced('metadata.exiftool.readTags')
+  async readTags(path: string): Promise<ImmichTags> {
+    try {
+      const args = mimeTypes.isVideo(path) ? ['-ee'] : [];
+      return (await this.exiftool.read(path, args)) as ImmichTags;
+    } catch (error: unknown) {
+      const span = trace.getActiveSpan();
+      span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      span?.recordException(error as Error);
+      this.logger.warn(`Error reading exif data (${path}): ${error}`);
      return {};
-    }) as Promise<ImmichTags>;
+    }
  }

-  extractBinaryTag(path: string, tagName: string): Promise<Buffer> {
+  @Traced('metadata.exiftool.extractBinaryTag')
+  async extractBinaryTag(path: string, tagName: string): Promise<Buffer> {
+    trace.getActiveSpan()?.setAttribute('exiftool.tagName', tagName);
    return this.exiftool.extractBinaryTagToBuffer(tagName, path);
  }

+  @Traced('metadata.exiftool.writeTags')
  async writeTags(path: string, tags: Partial<Tags>): Promise<void> {
    try {
      await this.exiftool.write(path, tags);
-    } catch (error) {
+    } catch (error: unknown) {
+      const span = trace.getActiveSpan();
+      span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      span?.recordException(error as Error);
      this.logger.warn(`Error writing exif data (${path}): ${error}`);
    }
  }
@@ -1,4 +1,5 @@
 import { Injectable } from '@nestjs/common';
+import { trace } from '@opentelemetry/api';
 import archiver from 'archiver';
 import chokidar, { ChokidarOptions } from 'chokidar';
 import { escapePath, glob, globStream } from 'fast-glob';
@@ -9,6 +10,7 @@ import { PassThrough, Readable, Writable } from 'node:stream';
 import { createGunzip, createGzip } from 'node:zlib';
 import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import { Traced } from 'src/utils/instrumentation';
 import { mimeTypes } from 'src/utils/mime-types';

 export interface WatchEvents {
@@ -106,9 +108,18 @@ export class StorageRepository {
    return createReadStream(filepath);
  }

+  @Traced('storage.createReadStream')
  async createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream> {
    const { size } = await fs.stat(filepath);
    await fs.access(filepath, constants.R_OK);
+
+    const span = trace.getActiveSpan();
+    span?.setAttribute('file.path', filepath);
+    span?.setAttribute('file.size', size);
+    if (mimeType) {
+      span?.setAttribute('file.content_type', mimeType);
+    }
+
    return {
      stream: createReadStream(filepath),
      length: size,
@@ -116,10 +127,15 @@ export class StorageRepository {
    };
  }

+  @Traced('storage.readFile')
  async readFile(filepath: string, options?: ReadOptionsWithBuffer<Buffer>): Promise<Buffer> {
+    const span = trace.getActiveSpan();
+    span?.setAttribute('file.path', filepath);
+
    const file = await fs.open(filepath);
    try {
-      const { buffer } = await file.read(options);
+      const { buffer, bytesRead } = await file.read(options);
+      span?.setAttribute('file.bytes_read', bytesRead);
      return buffer as Buffer;
    } finally {
      await file.close();
@@ -1,21 +1,11 @@
 import { Injectable } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
 import { MetricOptions } from '@opentelemetry/api';
-import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
-import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
-import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
-import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
-import { resourceFromAttributes } from '@opentelemetry/resources';
-import { AggregationType } from '@opentelemetry/sdk-metrics';
-import { NodeSDK, contextBase } from '@opentelemetry/sdk-node';
-import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import { contextBase } from '@opentelemetry/sdk-node';
 import { ClassConstructor } from 'class-transformer';
 import { snakeCase, startCase } from 'lodash';
 import { MetricService } from 'nestjs-otel';
 import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
-import { serverVersion } from 'src/constants';
 import { ImmichTelemetry, MetadataKey } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
@@ -51,49 +41,16 @@ export class MetricGroupRepository {
  }
 }

-const aggregationBoundaries = [
-  0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000,
-];
+// OpenTelemetry SDK is now initialized in telemetry-preload.ts
+// which runs before any modules are imported. This ensures the
+// global meter provider is available for nestjs-otel.

-let instance: NodeSDK | undefined;
-
-export const bootstrapTelemetry = (port: number) => {
-  if (instance) {
-    throw new Error('OpenTelemetry SDK already started');
-  }
-  instance = new NodeSDK({
-    resource: resourceFromAttributes({
-      [ATTR_SERVICE_NAME]: `immich`,
-      [ATTR_SERVICE_VERSION]: serverVersion.toString(),
-    }),
-    metricReader: new PrometheusExporter({ port }),
-    contextManager: new AsyncLocalStorageContextManager(),
-    instrumentations: [
-      new HttpInstrumentation(),
-      new IORedisInstrumentation(),
-      new NestInstrumentation(),
-      new PgInstrumentation(),
-    ],
-    views: [
-      {
-        instrumentName: '*',
-        instrumentUnit: 'ms',
-        aggregation: {
-          type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
-          options: { boundaries: aggregationBoundaries },
-        },
-      },
-    ],
-  });
-
-  instance.start();
+export const bootstrapTelemetry = (_port: number) => {
+  // No-op: SDK is initialized in telemetry-preload.ts
 };

 export const teardownTelemetry = async () => {
-  if (instance) {
-    await instance.shutdown();
-    instance = undefined;
-  }
+  // No-op: SDK shutdown is handled by process exit
 };

@Injectable()
@@ -8,6 +8,7 @@ import {
  AudioCodec,
  Colorspace,
  ExifOrientation,
+  GeneratedImageType,
  ImageFormat,
  JobName,
  JobStatus,
@@ -356,6 +357,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        expect.any(String),
      );
@@ -370,6 +372,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Thumbnail,
        },
        expect.any(String),
      );
@@ -581,6 +584,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        previewPath,
      );
@@ -595,6 +599,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Thumbnail,
        },
        thumbnailPath,
      );
@@ -630,6 +635,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        previewPath,
      );
@@ -644,6 +650,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Thumbnail,
        },
        thumbnailPath,
      );
@@ -838,6 +845,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        expect.any(String),
      );
@@ -870,6 +878,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Fullsize,
        },
        expect.any(String),
      );
@@ -884,6 +893,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        expect.any(String),
      );
@@ -914,6 +924,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Fullsize,
        },
        expect.any(String),
      );
@@ -928,6 +939,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Preview,
        },
        expect.any(String),
      );
@@ -959,6 +971,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Fullsize,
        },
        expect.any(String),
      );
@@ -1016,6 +1029,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Fullsize,
        },
        expect.any(String),
      );
@@ -1056,6 +1070,7 @@ describe(MediaService.name, () => {
          processInvalidImages: false,
          raw: rawInfo,
          edits: [],
+          imageType: GeneratedImageType.Fullsize,
        },
        expect.any(String),
      );
@@ -1300,6 +1315,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1345,6 +1361,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1388,6 +1405,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1431,6 +1449,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1474,6 +1493,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1517,6 +1537,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -1565,6 +1586,7 @@ describe(MediaService.name, () => {
          raw: info,
          processInvalidImages: false,
          size: 250,
+          imageType: GeneratedImageType.Person,
        },
        expect.any(String),
      );
@@ -12,6 +12,7 @@ import {
  AssetVisibility,
  AudioCodec,
  Colorspace,
+  GeneratedImageType,
  ImageFormat,
  JobName,
  JobStatus,
@@ -329,12 +330,12 @@ export class MediaService extends BaseService {
      this.mediaRepository.generateThumbhash(data, thumbnailOptions),
      this.mediaRepository.generateThumbnail(
        data,
-        { ...image.thumbnail, ...thumbnailOptions, edits: useEdits ? asset.edits : [] },
+        { ...image.thumbnail, ...thumbnailOptions, imageType: GeneratedImageType.Thumbnail, edits: useEdits ? asset.edits : [] },
        thumbnailPath,
      ),
      this.mediaRepository.generateThumbnail(
        data,
-        { ...image.preview, ...thumbnailOptions, edits: useEdits ? asset.edits : [] },
+        { ...image.preview, ...thumbnailOptions, imageType: GeneratedImageType.Preview, edits: useEdits ? asset.edits : [] },
        previewPath,
      ),
    ];
@@ -352,6 +353,7 @@ export class MediaService extends BaseService {
        format: image.fullsize.format,
        quality: image.fullsize.quality,
        progressive: image.fullsize.progressive,
+        imageType: GeneratedImageType.Fullsize,
        ...thumbnailOptions,
      };
      promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
@@ -438,6 +440,7 @@ export class MediaService extends BaseService {
      progressive: false,
      processInvalidImages: false,
      size: FACE_THUMBNAIL_SIZE,
+      imageType: GeneratedImageType.Person,
      edits: [
        {
          action: AssetEditAction.Crop,
@@ -0,0 +1,149 @@
+import '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import type { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { AggregationType, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import {
+  ATTR_VCS_REPOSITORY_REF_REVISION,
+  ATTR_VCS_REPOSITORY_URL_FULL,
+} from '@opentelemetry/semantic-conventions/incubating';
+import { isMainThread } from 'node:worker_threads';
+import { BullMQOtel } from 'bullmq-otel';
+
+export const bullMQTelemetry = new BullMQOtel('immich-jobs');
+
+const ignoredRoutes = new Set(['/api/server/ping']);
+const ignoredCallbacks = new Set(['pingServer']);
+
+class FilteringSpanProcessor implements SpanProcessor {
+  constructor(private delegate: SpanProcessor) {}
+
+  onStart(span: ReadableSpan, context: unknown): void {
+    (this.delegate.onStart as (span: ReadableSpan, context: unknown) => void)(span, context);
+  }
+
+  onEnd(span: ReadableSpan): void {
+    const route = span.attributes['http.route'] || span.attributes['http.target'];
+    if (typeof route === 'string' && ignoredRoutes.has(route)) {
+      return;
+    }
+
+    const callback = span.attributes['nestjs.callback'];
+    if (typeof callback === 'string' && ignoredCallbacks.has(callback)) {
+      return;
+    }
+
+    this.delegate.onEnd(span);
+  }
+
+  shutdown(): Promise<void> {
+    return this.delegate.shutdown();
+  }
+
+  forceFlush(): Promise<void> {
+    return this.delegate.forceFlush();
+  }
+}
+
+// Detect worker type to determine metrics port:
+// - Forked child (API worker): process.send exists, use IMMICH_API_METRICS_PORT
+// - Worker thread (microservices): !isMainThread, use IMMICH_MICROSERVICES_METRICS_PORT
+// - Main process: don't start metrics (avoid port conflicts with workers)
+const isForkedChild = typeof process.send === 'function';
+const isWorkerThread = !isMainThread;
+const isWorkerProcess = isForkedChild || isWorkerThread;
+
+const telemetryInclude = process.env.IMMICH_TELEMETRY_INCLUDE;
+const tracingEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+const metricsEnabled = telemetryInclude && telemetryInclude.length > 0;
+
+// Only start SDK in worker processes (not main/supervisor process)
+if (isWorkerProcess && (tracingEndpoint || metricsEnabled)) {
+  const workerType = isForkedChild ? 'api' : 'microservices';
+
+  const attributes: Record<string, string> = {
+    [ATTR_SERVICE_NAME]: `immich-${workerType}`,
+    [ATTR_SERVICE_VERSION]: process.env.npm_package_version || 'dev',
+  };
+
+  if (process.env.IMMICH_ENV) {
+    attributes['deployment.environment.name'] = process.env.IMMICH_ENV;
+  }
+
+  if (process.env.IMMICH_SOURCE_COMMIT) {
+    attributes[ATTR_VCS_REPOSITORY_REF_REVISION] = process.env.IMMICH_SOURCE_COMMIT;
+  }
+
+  if (process.env.IMMICH_REPOSITORY_URL) {
+    attributes[ATTR_VCS_REPOSITORY_URL_FULL] = process.env.IMMICH_REPOSITORY_URL;
+  }
+
+  if (process.env.IMMICH_BUILD) {
+    attributes['immich.build'] = process.env.IMMICH_BUILD;
+  }
+
+  if (process.env.IMMICH_SOURCE_REF) {
+    attributes['immich.source_ref'] = process.env.IMMICH_SOURCE_REF;
+  }
+
+  if (process.env.IMMICH_BUILD_IMAGE) {
+    attributes['immich.build_image'] = process.env.IMMICH_BUILD_IMAGE;
+  }
+
+  const resource = resourceFromAttributes(attributes);
+
+  const aggregationBoundaries = [
+    0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000,
+  ];
+
+  // Use appropriate port based on worker type
+  const defaultPort = isForkedChild ? '8081' : '8082';
+  const portEnvVar = isForkedChild ? 'IMMICH_API_METRICS_PORT' : 'IMMICH_MICROSERVICES_METRICS_PORT';
+  const metricsPort = Number.parseInt(process.env[portEnvVar] || defaultPort, 10);
+
+  const traceExporter = tracingEndpoint ? new OTLPTraceExporter({ url: `${tracingEndpoint}/v1/traces` }) : undefined;
+  const spanProcessor = traceExporter ? new FilteringSpanProcessor(new BatchSpanProcessor(traceExporter)) : undefined;
+
+  const metricReaders = [];
+  if (metricsEnabled) {
+    metricReaders.push(new PrometheusExporter({ port: metricsPort }));
+  }
+  if (tracingEndpoint) {
+    metricReaders.push(
+      new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter({ url: `${tracingEndpoint}/v1/metrics` }),
+      }),
+    );
+  }
+
+  const sdk = new NodeSDK({
+    resource,
+    spanProcessors: spanProcessor ? [spanProcessor] : undefined,
+    metricReaders: metricReaders.length > 0 ? metricReaders : undefined,
+    views: metricsEnabled
+      ? [
+          {
+            instrumentName: '*',
+            instrumentUnit: 'ms',
+            aggregation: {
+              type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
+              options: { boundaries: aggregationBoundaries },
+            },
+          },
+        ]
+      : undefined,
+    contextManager: new AsyncLocalStorageContextManager(),
+    instrumentations: [new HttpInstrumentation(), new IORedisInstrumentation(), new NestInstrumentation()],
+  });
+
+  sdk.start();
+}
@@ -10,6 +10,7 @@ import {
  AssetType,
  DatabaseSslMode,
  ExifOrientation,
+  GeneratedImageType,
  ImageFormat,
  JobName,
  MemoryType,
@@ -64,7 +65,10 @@ export interface DecodeToBufferOptions extends DecodeImageOptions {
  orientation?: ExifOrientation;
 }

-export type GenerateThumbnailOptions = Pick<ImageOptions, 'format' | 'quality' | 'progressive'> & DecodeToBufferOptions;
+export type GenerateThumbnailOptions = Pick<ImageOptions, 'format' | 'quality' | 'progressive'> &
+  DecodeToBufferOptions & {
+    imageType?: GeneratedImageType;
+  };

 export type GenerateThumbnailFromBufferOptions = GenerateThumbnailOptions & { raw: RawImageInfo };

@@ -178,8 +182,14 @@ export type ConcurrentQueueName = Exclude<
 export type Jobs = { [K in JobItem['name']]: (JobItem & { name: K })['data'] };
 export type JobOf<T extends JobName> = Jobs[T];

+export interface TraceContext {
+  traceparent: string;
+  tracestate?: string;
+}
+
 export interface IBaseJob {
  force?: boolean;
+  traceContext?: TraceContext;
 }

 export interface IDelayedJob extends IBaseJob {
@@ -1,6 +1,7 @@
 import {
  AliasedRawBuilder,
  DeduplicateJoinsPlugin,
+  Dialect,
  Expression,
  ExpressionBuilder,
  ExpressionWrapper,
@@ -22,6 +23,7 @@ import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } fr
 import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
 import { DB } from 'src/schema';
 import { DatabaseConnectionParams, VectorExtension } from 'src/types';
+import { createInstrumentedDriver } from 'src/utils/otel-kysely-plugin';

 type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object;

@@ -59,50 +61,65 @@ export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) =>
  };
 };

+const DATABASE_MAX_CONNECTIONS = 10;
+
+function createInstrumentedDialect(innerDialect: Dialect, maxConnections: number): Dialect {
+  return {
+    createAdapter: () => innerDialect.createAdapter(),
+    createDriver: () => createInstrumentedDriver(innerDialect.createDriver(), maxConnections),
+    createIntrospector: (db) => innerDialect.createIntrospector(db),
+    createQueryCompiler: () => innerDialect.createQueryCompiler(),
+  };
+}
+
 export const getKyselyConfig = (
  params: DatabaseConnectionParams,
  options: Partial<postgres.Options<Record<string, postgres.PostgresType>>> = {},
 ): KyselyConfig => {
  const config = asPostgresConnectionConfig(params);
+  const maxConnections = options.max ?? DATABASE_MAX_CONNECTIONS;
+
+  const innerDialect = new PostgresJSDialect({
+    postgres: postgres({
+      onnotice: (notice: Notice) => {
+        if (notice['severity'] !== 'NOTICE') {
+          console.warn('Postgres notice:', notice);
+        }
+      },
+      max: maxConnections,
+      types: {
+        date: {
+          to: 1184,
+          from: [1082, 1114, 1184],
+          serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x),
+          parse: (x: string) => new Date(x),
+        },
+        bigint: {
+          to: 20,
+          from: [20, 1700],
+          parse: (value: string) => Number.parseInt(value),
+          serialize: (value: number) => value.toString(),
+        },
+      },
+      connection: {
+        TimeZone: 'UTC',
+      },
+      host: config.host,
+      port: config.port,
+      username: config.username,
+      password: config.password,
+      database: config.database,
+      ssl: config.ssl,
+      ...options,
+    }),
+  });

  return {
-    dialect: new PostgresJSDialect({
-      postgres: postgres({
-        onnotice: (notice: Notice) => {
-          if (notice['severity'] !== 'NOTICE') {
-            console.warn('Postgres notice:', notice);
-          }
-        },
-        max: 10,
-        types: {
-          date: {
-            to: 1184,
-            from: [1082, 1114, 1184],
-            serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x),
-            parse: (x: string) => new Date(x),
-          },
-          bigint: {
-            to: 20,
-            from: [20, 1700],
-            parse: (value: string) => Number.parseInt(value),
-            serialize: (value: number) => value.toString(),
-          },
-        },
-        connection: {
-          TimeZone: 'UTC',
-        },
-        host: config.host,
-        port: config.port,
-        username: config.username,
-        password: config.password,
-        database: config.database,
-        ssl: config.ssl,
-        ...options,
-      }),
-    }),
-    log(event) {
+    dialect: createInstrumentedDialect(innerDialect, maxConnections),
+    plugins: [],
+    log: (event) => {
      if (event.level === 'error') {
-        console.error('Query failed :', {
+        console.error('Query failed:', {
          durationMs: event.queryDurationMillis,
          error: event.error,
          sql: event.query.sql,
@@ -1,13 +1,17 @@
 import { HttpException, StreamableFile } from '@nestjs/common';
+import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
 import { NextFunction, Response } from 'express';
-import { access, constants } from 'node:fs/promises';
+import { access, constants, stat } from 'node:fs/promises';
 import { basename, extname } from 'node:path';
+import { PassThrough } from 'node:stream';
 import { promisify } from 'node:util';
 import { CacheControl } from 'src/enum';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { ImmichReadStream } from 'src/repositories/storage.repository';
 import { isConnectionAborted } from 'src/utils/misc';

+const fileTracer = trace.getTracer('immich-file');
+
 export function getFileNameWithoutExtension(path: string): string {
  return basename(path, extname(path));
 }
@@ -49,6 +53,8 @@ export const sendFile = async (
  const _sendFile = (path: string, options: SendFileOptions) =>
    promisify<string, SendFileOptions>(res.sendFile).bind(res)(path, options);

+  const span = fileTracer.startSpan('file.send', { kind: SpanKind.INTERNAL }, context.active());
+
  try {
    const file = await handler();
    const cacheControlHeader = cacheControlHeaders[file.cacheControl];
@@ -64,10 +70,24 @@ export const sendFile = async (

    await access(file.path, constants.R_OK);

-    return await _sendFile(file.path, { dotfiles: 'allow' });
+    // Get file size for telemetry
+    const fileStats = await stat(file.path);
+
+    span.setAttribute('file.path', file.path);
+    span.setAttribute('file.size', fileStats.size);
+    span.setAttribute('file.content_type', file.contentType);
+    span.setAttribute('file.name', file.fileName || basename(file.path));
+    span.setStatus({ code: SpanStatusCode.OK });
+
+    const result = await _sendFile(file.path, { dotfiles: 'allow' });
+    span.end();
+    return result;
  } catch (error: Error | any) {
    // ignore client-closed connection
    if (isConnectionAborted(error) || res.headersSent) {
+      span.setAttribute('file.connection_aborted', true);
+      span.setStatus({ code: SpanStatusCode.OK });
+      span.end();
      return;
    }

@@ -76,11 +96,46 @@ export const sendFile = async (
      logger.error(`Unable to send file: ${error}`, error.stack);
    }

+    span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+    if (error instanceof Error) {
+      span.recordException(error);
+    }
+    span.end();
+
    res.header('Cache-Control', 'none');
    next(error);
  }
 };

 export const asStreamableFile = ({ stream, type, length }: ImmichReadStream) => {
-  return new StreamableFile(stream, { type, length });
+  const span = fileTracer.startSpan('file.stream', { kind: SpanKind.INTERNAL }, context.active());
+
+  if (length !== undefined) {
+    span.setAttribute('file.size', length);
+  }
+  if (type) {
+    span.setAttribute('file.content_type', type);
+  }
+
+  let bytesStreamed = 0;
+
+  const trackedStream = new PassThrough();
+  trackedStream.on('data', (chunk: Buffer) => {
+    bytesStreamed += chunk.length;
+  });
+  trackedStream.on('end', () => {
+    span.setAttribute('file.bytes_streamed', bytesStreamed);
+    span.setStatus({ code: SpanStatusCode.OK });
+    span.end();
+  });
+  trackedStream.on('error', (error: Error) => {
+    span.setAttribute('file.bytes_streamed', bytesStreamed);
+    span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+    span.recordException(error);
+    span.end();
+  });
+
+  stream.pipe(trackedStream);
+
+  return new StreamableFile(trackedStream, { type, length });
 };
@@ -0,0 +1,35 @@
+import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('immich');
+
+type TracedOptions = {
+  name: string;
+  kind?: SpanKind;
+};
+
+export function Traced(options: string | TracedOptions) {
+  const { name: spanName, kind = SpanKind.INTERNAL } =
+    typeof options === 'string' ? { name: options } : options;
+
+  return function (_target: unknown, _propertyKey: string, descriptor: PropertyDescriptor) {
+    const originalMethod = descriptor.value;
+
+    descriptor.value = async function (...args: unknown[]) {
+      return tracer.startActiveSpan(spanName, { kind }, context.active(), async (span: Span) => {
+        try {
+          const result = await originalMethod.apply(this, args);
+          span.setStatus({ code: SpanStatusCode.OK });
+          return result;
+        } catch (error) {
+          span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+          span.recordException(error as Error);
+          throw error;
+        } finally {
+          span.end();
+        }
+      });
+    };
+
+    return descriptor;
+  };
+}
@@ -0,0 +1,360 @@
+import { context, metrics, Span, SpanKind, SpanStatusCode, trace, ValueType } from '@opentelemetry/api';
+import type { CompiledQuery, DatabaseConnection, Driver, QueryResult, TransactionSettings } from 'kysely';
+
+const dbTracer = trace.getTracer('immich-db');
+const instrumentedConnections = new WeakSet<DatabaseConnection>();
+const traceDbConnections = process.env.IMMICH_OTEL_TRACE_DB_CONNECTIONS === 'true';
+
+let poolMetricsInitialized = false;
+
+function initializePoolMetrics(maxConnections: number) {
+  if (poolMetricsInitialized) {
+    return;
+  }
+  poolMetricsInitialized = true;
+
+  const meter = metrics.getMeter('immich-db');
+
+  meter
+    .createObservableGauge('db.client.connections.max', {
+      description: 'Maximum number of connections in the pool',
+      unit: '{connection}',
+      valueType: ValueType.INT,
+    })
+    .addCallback((observable) => {
+      observable.observe(maxConnections, { 'db.system': 'postgresql' });
+    });
+}
+
+function getOperationName(sql: string): string {
+  const trimmed = sql.trimStart().toUpperCase();
+  if (trimmed.startsWith('SELECT')) {
+    return 'SELECT';
+  }
+  if (trimmed.startsWith('INSERT')) {
+    return 'INSERT';
+  }
+  if (trimmed.startsWith('UPDATE')) {
+    return 'UPDATE';
+  }
+  if (trimmed.startsWith('DELETE')) {
+    return 'DELETE';
+  }
+  if (trimmed.startsWith('WITH')) {
+    return 'WITH';
+  }
+  if (trimmed.startsWith('BEGIN')) {
+    return 'BEGIN';
+  }
+  if (trimmed.startsWith('COMMIT')) {
+    return 'COMMIT';
+  }
+  if (trimmed.startsWith('ROLLBACK')) {
+    return 'ROLLBACK';
+  }
+  return 'QUERY';
+}
+
+function getTableName(sql: string): string | undefined {
+  const fromMatch = sql.match(/\bFROM\s+"?(\w+)"?/i);
+  if (fromMatch) {
+    return fromMatch[1];
+  }
+
+  const intoMatch = sql.match(/\bINTO\s+"?(\w+)"?/i);
+  if (intoMatch) {
+    return intoMatch[1];
+  }
+
+  const updateMatch = sql.match(/\bUPDATE\s+"?(\w+)"?/i);
+  if (updateMatch) {
+    return updateMatch[1];
+  }
+
+  return undefined;
+}
+
+function createInstrumentedConnection(
+  connection: DatabaseConnection,
+  transactionSpan?: Span,
+): DatabaseConnection {
+  if (instrumentedConnections.has(connection)) {
+    return connection;
+  }
+
+  const instrumentedConnection: DatabaseConnection = {
+    async executeQuery<R>(compiledQuery: CompiledQuery): Promise<QueryResult<R>> {
+      const sql = compiledQuery.sql;
+      const operationName = getOperationName(sql);
+      const tableName = getTableName(sql);
+      const spanName = tableName ? `${operationName} ${tableName}` : operationName;
+
+      const parentContext = transactionSpan
+        ? trace.setSpan(context.active(), transactionSpan)
+        : context.active();
+
+      return dbTracer.startActiveSpan(spanName, { kind: SpanKind.CLIENT }, parentContext, async (span) => {
+        try {
+          span.setAttribute('db.system', 'postgresql');
+          span.setAttribute('db.operation.name', operationName);
+          span.setAttribute('db.query.text', sql);
+
+          if (tableName) {
+            span.setAttribute('db.collection.name', tableName);
+          }
+
+          if (compiledQuery.parameters.length > 0) {
+            span.setAttribute('db.query.parameter_count', compiledQuery.parameters.length);
+          }
+
+          const result = await connection.executeQuery<R>(compiledQuery);
+
+          span.setAttribute('db.response.row_count', result.rows.length);
+
+          if (result.numAffectedRows !== undefined) {
+            span.setAttribute('db.operation.affected_rows', Number(result.numAffectedRows));
+          }
+
+          span.setStatus({ code: SpanStatusCode.OK });
+          return result;
+        } catch (error) {
+          span.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: error instanceof Error ? error.message : String(error),
+          });
+          if (error instanceof Error) {
+            span.recordException(error);
+          }
+          throw error;
+        } finally {
+          span.end();
+        }
+      });
+    },
+
+    async *streamQuery<R>(
+      compiledQuery: CompiledQuery,
+      chunkSize: number,
+    ): AsyncIterableIterator<QueryResult<R>> {
+      const sql = compiledQuery.sql;
+      const operationName = getOperationName(sql);
+      const tableName = getTableName(sql);
+      const spanName = tableName ? `STREAM ${operationName} ${tableName}` : `STREAM ${operationName}`;
+
+      const parentContext = transactionSpan
+        ? trace.setSpan(context.active(), transactionSpan)
+        : context.active();
+
+      const span = dbTracer.startSpan(spanName, { kind: SpanKind.CLIENT }, parentContext);
+
+      span.setAttribute('db.system', 'postgresql');
+      span.setAttribute('db.operation.name', operationName);
+      span.setAttribute('db.query.text', sql);
+      span.setAttribute('db.stream.chunk_size', chunkSize);
+
+      if (tableName) {
+        span.setAttribute('db.collection.name', tableName);
+      }
+
+      let totalRows = 0;
+      let chunkCount = 0;
+
+      try {
+        for await (const chunk of connection.streamQuery<R>(compiledQuery, chunkSize)) {
+          chunkCount++;
+          totalRows += chunk.rows.length;
+          yield chunk;
+        }
+
+        span.setAttribute('db.stream.chunk_count', chunkCount);
+        span.setAttribute('db.stream.total_rows', totalRows);
+        span.setStatus({ code: SpanStatusCode.OK });
+      } catch (error) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: error instanceof Error ? error.message : String(error),
+        });
+        if (error instanceof Error) {
+          span.recordException(error);
+        }
+        throw error;
+      } finally {
+        span.end();
+      }
+    },
+  };
+
+  instrumentedConnections.add(instrumentedConnection);
+  return instrumentedConnection;
+}
+
+const transactionSpans = new WeakMap<DatabaseConnection, Span>();
+const unwrappedConnections = new WeakMap<DatabaseConnection, DatabaseConnection>();
+
+/**
+ * Creates an instrumented driver that wraps a Kysely driver with OpenTelemetry tracing.
+ * Provides:
+ * - Query spans with row counts and affected rows
+ * - Streaming query spans with chunk metrics
+ * - Transaction spans that parent query spans
+ * - Connection acquisition timing
+ */
+export function createInstrumentedDriver(driver: Driver, maxConnections: number): Driver {
+  initializePoolMetrics(maxConnections);
+
+  return new Proxy(driver, {
+    get(target, prop, receiver) {
+      if (prop === 'init' || prop === 'destroy') {
+        const method = Reflect.get(target, prop, target);
+        if (typeof method === 'function') {
+          return method.bind(target);
+        }
+        return method;
+      }
+
+      if (prop === 'acquireConnection') {
+        return async function (): Promise<DatabaseConnection> {
+          if (!traceDbConnections) {
+            const connection = await target.acquireConnection();
+            const instrumentedConnection = createInstrumentedConnection(connection);
+            unwrappedConnections.set(instrumentedConnection, connection);
+            return instrumentedConnection;
+          }
+
+          return dbTracer.startActiveSpan(
+            'DB Connection Acquire',
+            { kind: SpanKind.CLIENT },
+            context.active(),
+            async (span) => {
+              try {
+                span.setAttribute('db.system', 'postgresql');
+
+                const connection = await target.acquireConnection();
+                const instrumentedConnection = createInstrumentedConnection(connection);
+
+                unwrappedConnections.set(instrumentedConnection, connection);
+
+                span.setStatus({ code: SpanStatusCode.OK });
+                return instrumentedConnection;
+              } catch (error) {
+                span.setStatus({
+                  code: SpanStatusCode.ERROR,
+                  message: error instanceof Error ? error.message : String(error),
+                });
+                if (error instanceof Error) {
+                  span.recordException(error);
+                }
+                throw error;
+              } finally {
+                span.end();
+              }
+            },
+          );
+        };
+      }
+
+      if (prop === 'beginTransaction') {
+        return async function (
+          connection: DatabaseConnection,
+          settings: TransactionSettings,
+        ): Promise<void> {
+          const unwrapped = unwrappedConnections.get(connection) ?? connection;
+
+          const transactionSpan = dbTracer.startSpan(
+            'DB Transaction',
+            { kind: SpanKind.CLIENT },
+            context.active(),
+          );
+          transactionSpan.setAttribute('db.system', 'postgresql');
+          transactionSpan.setAttribute('db.operation.name', 'TRANSACTION');
+
+          if (settings.isolationLevel) {
+            transactionSpan.setAttribute('db.transaction.isolation_level', settings.isolationLevel);
+          }
+
+          transactionSpans.set(connection, transactionSpan);
+
+          const instrumentedForTransaction = createInstrumentedConnection(unwrapped, transactionSpan);
+          unwrappedConnections.set(instrumentedForTransaction, unwrapped);
+
+          Object.assign(connection, instrumentedForTransaction);
+
+          try {
+            await target.beginTransaction(unwrapped, settings);
+          } catch (error) {
+            transactionSpan.setStatus({
+              code: SpanStatusCode.ERROR,
+              message: error instanceof Error ? error.message : String(error),
+            });
+            if (error instanceof Error) {
+              transactionSpan.recordException(error);
+            }
+            transactionSpan.end();
+            transactionSpans.delete(connection);
+            throw error;
+          }
+        };
+      }
+
+      if (prop === 'commitTransaction') {
+        return async function (connection: DatabaseConnection): Promise<void> {
+          const unwrapped = unwrappedConnections.get(connection) ?? connection;
+          const transactionSpan = transactionSpans.get(connection);
+
+          try {
+            await target.commitTransaction(unwrapped);
+            transactionSpan?.setStatus({ code: SpanStatusCode.OK });
+          } catch (error) {
+            transactionSpan?.setStatus({
+              code: SpanStatusCode.ERROR,
+              message: error instanceof Error ? error.message : String(error),
+            });
+            if (error instanceof Error) {
+              transactionSpan?.recordException(error);
+            }
+            throw error;
+          } finally {
+            transactionSpan?.end();
+            transactionSpans.delete(connection);
+          }
+        };
+      }
+
+      if (prop === 'rollbackTransaction') {
+        return async function (connection: DatabaseConnection): Promise<void> {
+          const unwrapped = unwrappedConnections.get(connection) ?? connection;
+          const transactionSpan = transactionSpans.get(connection);
+
+          try {
+            await target.rollbackTransaction(unwrapped);
+            transactionSpan?.setAttribute('db.transaction.rollback', true);
+            transactionSpan?.setStatus({ code: SpanStatusCode.OK });
+          } catch (error) {
+            transactionSpan?.setStatus({
+              code: SpanStatusCode.ERROR,
+              message: error instanceof Error ? error.message : String(error),
+            });
+            if (error instanceof Error) {
+              transactionSpan?.recordException(error);
+            }
+            throw error;
+          } finally {
+            transactionSpan?.end();
+            transactionSpans.delete(connection);
+          }
+        };
+      }
+
+      if (prop === 'releaseConnection') {
+        return async function (connection: DatabaseConnection): Promise<void> {
+          const unwrapped = unwrappedConnections.get(connection) ?? connection;
+          unwrappedConnections.delete(connection);
+          transactionSpans.delete(connection);
+          return target.releaseConnection(unwrapped);
+        };
+      }
+
+      return Reflect.get(target, prop, receiver);
+    },
+  });
+}
@@ -1,3 +1,6 @@
+// Must be first import - sets up OpenTelemetry SDK before any modules load
+import 'src/telemetry-preload';
+
 import { NestFactory } from '@nestjs/core';
 import { isMainThread } from 'node:worker_threads';
 import { MicroservicesModule } from 'src/app.module';