From e03ed9b3873f88cdc16857b89e2e162abed47169 Mon Sep 17 00:00:00 2001 From: midzelis Date: Tue, 27 Jan 2026 16:54:23 +0000 Subject: [PATCH] feat(server): add OpenTelemetry tracing and metrics support Adds comprehensive OpenTelemetry instrumentation for better observability: Tracing: - HTTP requests via @opentelemetry/instrumentation-http - NestJS controllers via @opentelemetry/instrumentation-nestjs-core - Redis operations via @opentelemetry/instrumentation-ioredis - BullMQ job queues via bullmq-otel - Database queries via Kysely log callback with accurate timing - File operations (send, stream, read) - Media processing (thumbnails, thumbhash, EXIF operations) - Metadata extraction Metrics: - Database connection pool usage (used/idle connections) The SDK is initialized in telemetry-preload.ts which is preloaded via --require before the main app starts. This ensures http instrumentation hooks are in place before any http module is imported. Enable tracing by setting OTEL_EXPORTER_OTLP_ENDPOINT environment variable. Also adds TraceContext to IBaseJob for future distributed trace propagation across BullMQ job boundaries. --- docker/signoz-immich-dashboard.json | 1875 +++++++++++++++++ pnpm-lock.yaml | 479 ++++- server/bin/immich-dev | 13 + server/bin/start.sh | 2 +- server/package.json | 5 +- server/src/enum.ts | 7 + server/src/repositories/config.repository.ts | 2 + .../machine-learning.repository.ts | 89 +- server/src/repositories/media.repository.ts | 206 +- .../src/repositories/metadata.repository.ts | 28 +- server/src/repositories/storage.repository.ts | 18 +- .../src/repositories/telemetry.repository.ts | 57 +- server/src/services/media.service.spec.ts | 22 + server/src/services/media.service.ts | 7 +- server/src/telemetry-preload.ts | 149 ++ server/src/types.ts | 12 +- server/src/utils/database.ts | 89 +- server/src/utils/file.ts | 61 +- server/src/utils/instrumentation.ts | 35 + server/src/utils/otel-kysely-plugin.ts | 360 ++++ server/src/workers/microservices.ts | 3 + 21 files changed, 3270 insertions(+), 249 deletions(-) create mode 100644 docker/signoz-immich-dashboard.json create mode 100644 server/src/telemetry-preload.ts create mode 100644 server/src/utils/instrumentation.ts create mode 100644 server/src/utils/otel-kysely-plugin.ts diff --git a/docker/signoz-immich-dashboard.json b/docker/signoz-immich-dashboard.json new file mode 100644 index 0000000000..35b0e2978a --- /dev/null +++ b/docker/signoz-immich-dashboard.json @@ -0,0 +1,1875 @@ +{ + "description": "Comprehensive monitoring dashboard for Immich photo sharing server with OpenTelemetry instrumentation", + "layout": [ + {"h": 1, "i": "row-overview", "maxH": 1, "minH": 1, "minW": 12, "moved": false, "static": false, "w": 12, "x": 0, "y": 0}, + {"h": 6, "i": "http-latency-p99", "moved": false, "static": false, "w": 4, "x": 0, "y": 1}, + {"h": 6, "i": "http-throughput", "moved": false, "static": false, "w": 4, "x": 4, "y": 1}, + {"h": 6, "i": "http-error-rate", "moved": false, "static": false, "w": 4, "x": 8, "y": 1}, + {"h": 6, "i": "http-status-codes", "moved": false, "static": false, "w": 6, "x": 0, "y": 7}, + {"h": 6, "i": "top-endpoints", "moved": false, "static": false, "w": 6, "x": 6, "y": 7}, + {"h": 1, "i": "row-database", "maxH": 1, "minH": 1, "minW": 12, "moved": false, "static": false, "w": 12, "x": 0, "y": 13}, + {"h": 6, "i": "db-query-latency", "moved": false, "static": false, "w": 6, "x": 0, "y": 14}, + {"h": 6, "i": "db-operations", "moved": false, "static": false, "w": 6, "x": 6, "y": 14}, + {"h": 6, "i": "db-connections", "moved": false, "static": false, "w": 6, "x": 0, "y": 20}, + {"h": 6, "i": "db-slow-queries", "moved": false, "static": false, "w": 6, "x": 6, "y": 20}, + {"h": 1, "i": "row-jobs", "maxH": 1, "minH": 1, "minW": 12, "moved": false, "static": false, "w": 12, "x": 0, "y": 26}, + {"h": 6, "i": "job-throughput", "moved": false, "static": false, "w": 6, "x": 0, "y": 27}, + {"h": 6, "i": "job-latency", "moved": false, "static": false, "w": 6, "x": 6, "y": 27}, + {"h": 1, "i": "row-redis", "maxH": 1, "minH": 1, "minW": 12, "moved": false, "static": false, "w": 12, "x": 0, "y": 33}, + {"h": 6, "i": "redis-operations", "moved": false, "static": false, "w": 6, "x": 0, "y": 34}, + {"h": 6, "i": "redis-latency", "moved": false, "static": false, "w": 6, "x": 6, "y": 34} + ], + "panelMap": { + "row-overview": { + "collapsed": false, + "widgets": [ + {"h": 6, "i": "http-latency-p99", "moved": false, "static": false, "w": 4, "x": 0, "y": 1}, + {"h": 6, "i": "http-throughput", "moved": false, "static": false, "w": 4, "x": 4, "y": 1}, + {"h": 6, "i": "http-error-rate", "moved": false, "static": false, "w": 4, "x": 8, "y": 1}, + {"h": 6, "i": "http-status-codes", "moved": false, "static": false, "w": 6, "x": 0, "y": 7}, + {"h": 6, "i": "top-endpoints", "moved": false, "static": false, "w": 6, "x": 6, "y": 7} + ] + }, + "row-database": { + "collapsed": false, + "widgets": [ + {"h": 6, "i": "db-query-latency", "moved": false, "static": false, "w": 6, "x": 0, "y": 14}, + {"h": 6, "i": "db-operations", "moved": false, "static": false, "w": 6, "x": 6, "y": 14}, + {"h": 6, "i": "db-connections", "moved": false, "static": false, "w": 6, "x": 0, "y": 20}, + {"h": 6, "i": "db-slow-queries", "moved": false, "static": false, "w": 6, "x": 6, "y": 20} + ] + }, + "row-jobs": { + "collapsed": false, + "widgets": [ + {"h": 6, "i": "job-throughput", "moved": false, "static": false, "w": 6, "x": 0, "y": 27}, + {"h": 6, "i": "job-latency", "moved": false, "static": false, "w": 6, "x": 6, "y": 27} + ] + }, + "row-redis": { + "collapsed": false, + "widgets": [ + {"h": 6, "i": "redis-operations", "moved": false, "static": false, "w": 6, "x": 0, "y": 34}, + {"h": 6, "i": "redis-latency", "moved": false, "static": false, "w": 6, "x": 6, "y": 34} + ] + } + }, + "tags": ["immich", "photo-sharing", "apm", "database", "redis", "jobs"], + "title": "Immich Photo Server", + "uploadedGrafana": false, + "variables": {}, + "version": "v4", + "widgets": [ + { + "description": "", + "id": "row-overview", + "panelTypes": "row", + "title": "HTTP API Overview" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "P99 latency of HTTP requests to the Immich API", + "fillSpans": false, + "id": "http-latency-p99", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p99", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "p99", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "p99", + "stepInterval": 60, + "timeAggregation": "p99" + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p90", + "dataSource": "traces", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "filter-service-b", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind-b", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "p90", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "avg", + "spaceAggregation": "p90", + "stepInterval": 60, + "timeAggregation": "p90" + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p50", + "dataSource": "traces", + "disabled": false, + "expression": "C", + "filters": { + "items": [ + { + "id": "filter-service-c", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind-c", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "p50", + "limit": null, + "orderBy": [], + "queryName": "C", + "reduceTo": "avg", + "spaceAggregation": "p50", + "stepInterval": 60, + "timeAggregation": "p50" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/1000000", + "legend": "p99 (ms)", + "queryName": "F1" + }, + { + "disabled": false, + "expression": "B/1000000", + "legend": "p90 (ms)", + "queryName": "F2" + }, + { + "disabled": false, + "expression": "C/1000000", + "legend": "p50 (ms)", + "queryName": "F3" + } + ] + }, + "clickhouse_sql": [], + "id": "http-latency-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "API Latency", + "yAxisUnit": "ms" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "HTTP requests per second to the Immich API", + "fillSpans": false, + "id": "http-throughput", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "Requests/sec", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "http-throughput-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Request Throughput", + "yAxisUnit": "reqps" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Percentage of HTTP requests resulting in errors", + "fillSpans": false, + "id": "http-error-rate", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + }, + { + "id": "filter-error", + "key": { + "dataType": "bool", + "id": "hasError--bool----true", + "isColumn": true, + "isJSON": false, + "key": "hasError", + "type": "" + }, + "op": "=", + "value": "true" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "Errors", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + }, + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "filter-service-b", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind-b", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "Total", + "limit": null, + "orderBy": [], + "queryName": "B", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/B*100", + "legend": "Error Rate %", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "http-error-rate-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [ + {"color": "#f5222d", "index": 0, "value": 5} + ], + "timePreferance": "GLOBAL_TIME", + "title": "Error Rate", + "yAxisUnit": "percent" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Distribution of HTTP response status codes", + "fillSpans": true, + "id": "http-status-codes", + "isStacked": true, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "0.7", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "int64", + "id": "http.status_code--int64--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.status_code", + "type": "tag" + } + ], + "having": [], + "legend": "{{http.status_code}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "http-status-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "HTTP Status Codes", + "yAxisUnit": "reqps" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Top API endpoints by request count and average latency", + "fillSpans": false, + "id": "top-endpoints", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + }, + { + "id": "filter-target", + "key": { + "dataType": "string", + "id": "http.target--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.target", + "type": "tag" + }, + "op": "exists", + "value": "" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "http.target--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.target", + "type": "tag" + }, + { + "dataType": "string", + "id": "http.method--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.method", + "type": "tag" + } + ], + "having": [], + "legend": "", + "limit": 20, + "orderBy": [{"columnName": "A", "order": "desc"}], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "count" + }, + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "avg", + "dataSource": "traces", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "filter-service-b", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-kind-b", + "key": { + "dataType": "int64", + "id": "kind--int64----true", + "isColumn": true, + "isJSON": false, + "key": "kind", + "type": "" + }, + "op": "=", + "value": "2" + }, + { + "id": "filter-target-b", + "key": { + "dataType": "string", + "id": "http.target--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.target", + "type": "tag" + }, + "op": "exists", + "value": "" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "http.target--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.target", + "type": "tag" + }, + { + "dataType": "string", + "id": "http.method--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.method", + "type": "tag" + } + ], + "having": [], + "legend": "", + "limit": 20, + "orderBy": [], + "queryName": "B", + "reduceTo": "avg", + "spaceAggregation": "avg", + "stepInterval": 60, + "timeAggregation": "avg" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "B/1000000", + "legend": "Avg Latency (ms)", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "top-endpoints-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "id": "serviceName--string--resource--false", + "isColumn": false, + "isJSON": false, + "key": "serviceName", + "type": "resource" + }, + { + "dataType": "string", + "id": "body--string--body--true", + "isColumn": true, + "isJSON": false, + "key": "body", + "type": "body" + }, + { + "dataType": "string", + "id": "severity_text--string----true", + "isColumn": true, + "isJSON": false, + "key": "severity_text", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "http.target--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.target", + "type": "tag" + }, + { + "dataType": "string", + "id": "http.method--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "http.method", + "type": "tag" + } + ], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Top API Endpoints", + "yAxisUnit": "short" + }, + { + "description": "", + "id": "row-database", + "panelTypes": "row", + "title": "Database (PostgreSQL)" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Database query latency by operation type (SELECT, INSERT, UPDATE, DELETE)", + "fillSpans": false, + "id": "db-query-latency", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p95", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-db", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "postgresql" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "db.operation.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.operation.name", + "type": "tag" + } + ], + "having": [], + "legend": "{{db.operation.name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "p95", + "stepInterval": 60, + "timeAggregation": "p95" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/1000000", + "legend": "{{db.operation.name}} P95", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "db-latency-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "DB Query Latency (P95)", + "yAxisUnit": "ms" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Database operations per second by type", + "fillSpans": true, + "id": "db-operations", + "isStacked": true, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "0.7", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-db", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "postgresql" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "db.operation.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.operation.name", + "type": "tag" + } + ], + "having": [], + "legend": "{{db.operation.name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "db-ops-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "DB Operations/sec", + "yAxisUnit": "ops" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Database query distribution by table", + "fillSpans": true, + "id": "db-connections", + "isStacked": true, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "0.7", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-db", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "postgresql" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "db.collection.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.collection.name", + "type": "tag" + } + ], + "having": [], + "legend": "{{db.collection.name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "db-connections-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "DB Queries by Table", + "yAxisUnit": "ops" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {"F1": "ms"}, + "description": "Slowest database queries by table", + "fillSpans": false, + "id": "db-slow-queries", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p99", + "dataSource": "traces", + "disabled": true, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-db", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "postgresql" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "db.collection.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.collection.name", + "type": "tag" + }, + { + "dataType": "string", + "id": "db.operation.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.operation.name", + "type": "tag" + } + ], + "having": [], + "legend": "", + "limit": 15, + "orderBy": [{"columnName": "F1", "order": "desc"}], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "p99", + "stepInterval": 60, + "timeAggregation": "p99" + }, + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "B", + "filters": { + "items": [ + { + "id": "filter-service-b", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-db-b", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "postgresql" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "db.collection.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.collection.name", + "type": "tag" + }, + { + "dataType": "string", + "id": "db.operation.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.operation.name", + "type": "tag" + } + ], + "having": [], + "legend": "", + "limit": 15, + "orderBy": [], + "queryName": "B", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "count" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/1000000", + "legend": "P99 Latency (ms)", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "db-slow-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "id": "serviceName--string--resource--false", + "isColumn": false, + "isJSON": false, + "key": "serviceName", + "type": "resource" + }, + { + "dataType": "string", + "id": "body--string--body--true", + "isColumn": true, + "isJSON": false, + "key": "body", + "type": "body" + }, + { + "dataType": "string", + "id": "severity_text--string----true", + "isColumn": true, + "isJSON": false, + "key": "severity_text", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + { + "dataType": "string", + "id": "db.collection.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.collection.name", + "type": "tag" + }, + { + "dataType": "string", + "id": "db.operation.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.operation.name", + "type": "tag" + } + ], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Slowest DB Queries by Table", + "yAxisUnit": "short" + }, + { + "description": "", + "id": "row-jobs", + "panelTypes": "row", + "title": "Background Jobs (BullMQ)" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Background job processing rate by queue", + "fillSpans": true, + "id": "job-throughput", + "isStacked": true, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "0.7", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-bullmq", + "key": { + "dataType": "string", + "id": "bullmq.queue.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "bullmq.queue.name", + "type": "tag" + }, + "op": "exists", + "value": "" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "bullmq.queue.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "bullmq.queue.name", + "type": "tag" + } + ], + "having": [], + "legend": "{{bullmq.queue.name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "job-throughput-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Job Throughput by Queue", + "yAxisUnit": "ops" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Background job processing latency (P95) by queue", + "fillSpans": false, + "id": "job-latency", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p95", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "=", + "value": "immich" + }, + { + "id": "filter-bullmq", + "key": { + "dataType": "string", + "id": "bullmq.queue.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "bullmq.queue.name", + "type": "tag" + }, + "op": "exists", + "value": "" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "bullmq.queue.name--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "bullmq.queue.name", + "type": "tag" + } + ], + "having": [], + "legend": "{{bullmq.queue.name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "p95", + "stepInterval": 60, + "timeAggregation": "p95" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/1000000", + "legend": "{{bullmq.queue.name}} P95", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "job-latency-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Job Latency (P95)", + "yAxisUnit": "ms" + }, + { + "description": "", + "id": "row-redis", + "panelTypes": "row", + "title": "Redis Cache" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Redis operations per second by command", + "fillSpans": true, + "id": "redis-operations", + "isStacked": true, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "0.7", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-redis", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "redis" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "name--string----true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "" + } + ], + "having": [], + "legend": "{{name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [], + "id": "redis-ops-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Redis Operations/sec", + "yAxisUnit": "ops" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "Redis command latency (P95) by operation", + "fillSpans": false, + "id": "redis-latency", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "graph", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "float64", + "id": "durationNano--float64----true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "" + }, + "aggregateOperator": "p95", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [ + { + "id": "filter-service", + "key": { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + "op": "contains", + "value": "immich" + }, + { + "id": "filter-redis", + "key": { + "dataType": "string", + "id": "db.system--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "db.system", + "type": "tag" + }, + "op": "=", + "value": "redis" + } + ], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "name--string----true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "" + } + ], + "having": [], + "legend": "{{name}}", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "p95", + "stepInterval": 60, + "timeAggregation": "p95" + } + ], + "queryFormulas": [ + { + "disabled": false, + "expression": "A/1000000", + "legend": "{{name}} P95", + "queryName": "F1" + } + ] + }, + "clickhouse_sql": [], + "id": "redis-latency-query", + "promql": [], + "queryType": "builder" + }, + "selectedLogFields": [], + "selectedTracesFields": [], + "softMax": null, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Redis Latency (P95)", + "yAxisUnit": "ms" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6de0038411..b87588c5a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -373,9 +373,15 @@ importers: '@opentelemetry/context-async-hooks': specifier: ^2.0.0 version: 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http': + specifier: ^0.211.0 + version: 0.211.0(@opentelemetry/api@1.9.0) '@opentelemetry/exporter-prometheus': specifier: ^0.210.0 version: 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http': + specifier: ^0.211.0 + version: 0.211.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-http': specifier: ^0.210.0 version: 0.210.0(@opentelemetry/api@1.9.0) @@ -385,9 +391,6 @@ importers: '@opentelemetry/instrumentation-nestjs-core': specifier: ^0.56.0 version: 0.56.0(@opentelemetry/api@1.9.0) - '@opentelemetry/instrumentation-pg': - specifier: ^0.62.0 - version: 0.62.0(@opentelemetry/api@1.9.0) '@opentelemetry/resources': specifier: ^2.0.1 version: 2.4.0(@opentelemetry/api@1.9.0) @@ -397,6 +400,9 @@ importers: '@opentelemetry/sdk-node': specifier: ^0.210.0 version: 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': + specifier: ^2.5.0 + version: 2.5.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': specifier: ^1.34.0 version: 1.38.0 @@ -427,6 +433,9 @@ importers: bullmq: specifier: ^5.51.0 version: 5.66.5 + bullmq-otel: + specifier: ^1.1.0 + version: 1.1.0 chokidar: specifier: ^4.0.3 version: 4.0.3 @@ -670,19 +679,19 @@ importers: version: 13.15.10 '@vitest/coverage-v8': specifier: ^3.0.0 - version: 3.2.4(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@2.6.1)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + version: 3.2.4(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@1.21.7)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) eslint: specifier: ^9.14.0 - version: 9.39.2(jiti@2.6.1) + version: 9.39.2(jiti@1.21.7) eslint-config-prettier: specifier: ^10.1.8 - version: 10.1.8(eslint@9.39.2(jiti@2.6.1)) + version: 10.1.8(eslint@9.39.2(jiti@1.21.7)) eslint-plugin-prettier: specifier: ^5.1.3 - version: 5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(prettier@3.8.0) + version: 5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7))(prettier@3.8.0) eslint-plugin-unicorn: specifier: ^62.0.0 - version: 62.0.0(eslint@9.39.2(jiti@2.6.1)) + version: 62.0.0(eslint@9.39.2(jiti@1.21.7)) globals: specifier: ^16.0.0 version: 16.5.0 @@ -718,16 +727,16 @@ importers: version: 5.9.3 typescript-eslint: specifier: ^8.28.0 - version: 8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + version: 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) unplugin-swc: specifier: ^1.4.5 version: 1.5.9(@swc/core@1.15.8(@swc/helpers@0.5.17))(rollup@4.55.1) vite-tsconfig-paths: specifier: ^6.0.0 - version: 6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.10.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + version: 6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@2.6.1)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@1.21.7)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) web: dependencies: @@ -3839,6 +3848,10 @@ packages: resolution: {integrity: sha512-CMtLxp+lYDriveZejpBND/2TmadrrhUfChyxzmkFtHaMDdSKfP59MAYyA0ICBvEBdm3iXwLcaj/8Ic/pnGw9Yg==} engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.211.0': + resolution: {integrity: sha512-swFdZq8MCdmdR22jTVGQDhwqDzcI4M10nhjXkLr1EsIzXgZBqm4ZlmmcWsg3TSNf+3mzgOiqveXmBLZuDi2Lgg==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} @@ -3861,6 +3874,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@2.5.0': + resolution: {integrity: sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/exporter-logs-otlp-grpc@0.210.0': resolution: {integrity: sha512-+BolenqOO6ow65go7uWRYPvvs/BBIWp1mtRn93VvGduqvMVH/IY8nXrt80a4L9hZ7lHi2Tq2/NcC3H2QzcWKag==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3891,6 +3910,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/exporter-metrics-otlp-http@0.211.0': + resolution: {integrity: sha512-lfHXElPAoDSPpPO59DJdN5FLUnwi1wxluLTWQDayqrSPfWRnluzxRhD+g7rF8wbj1qCz0sdqABl//ug1IZyWvA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/exporter-metrics-otlp-proto@0.210.0': resolution: {integrity: sha512-CFa7SOinYOVWIWJuQL7XFeyedzmFGIpHpSMNFE8Xefb6iGB4m+MukQecdssvPcJKYlfF5FpovEOLXwafAzsXWQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3915,6 +3940,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/exporter-trace-otlp-http@0.211.0': + resolution: {integrity: sha512-F1Rv3JeMkgS//xdVjbQMrI3+26e5SXC7vXA6trx8SWEA0OUhw4JHB+qeHtH0fJn46eFItrYbL5m8j4qi9Sfaxw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/exporter-trace-otlp-proto@0.210.0': resolution: {integrity: sha512-qVUY7Hsm/t5buGOtPcTV1Ch4W9kj2wGaQaAF5FO4XR8TMKl2GM45tUCnr0/1dF3wo4RG9khMxrddeQWdRL4fIg==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3951,12 +3982,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/instrumentation-pg@0.62.0': - resolution: {integrity: sha512-/ZSMRCyFRMjQVx7Wf+BIAOMEdN/XWBbAGTNLKfQgGYs1GlmdiIFkUy8Z8XGkToMpKrgZju0drlTQpqt4Ul7R6w==} - engines: {node: ^18.19.0 || >=20.6.0} - peerDependencies: - '@opentelemetry/api': ^1.3.0 - '@opentelemetry/instrumentation@0.210.0': resolution: {integrity: sha512-sLMhyHmW9katVaLUOKpfCnxSGhZq2t1ReWgwsu2cSgxmDVMB690H9TanuexanpFI94PJaokrqbp8u9KYZDUT5g==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3969,6 +3994,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-exporter-base@0.211.0': + resolution: {integrity: sha512-bp1+63V8WPV+bRI9EQG6E9YID1LIHYSZVbp7f+44g9tRzCq+rtw/o4fpL5PC31adcUsFiz/oN0MdLISSrZDdrg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-grpc-exporter-base@0.210.0': resolution: {integrity: sha512-fEJs8UhkFMrdXMOCLXyKd2uc6N209tIi8IBNqSTi83ri+MlMFrBKnOtklmv9/zzxovoN5zD1waRt6XBFGPfmIw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -3981,6 +4012,12 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-transformer@0.211.0': + resolution: {integrity: sha512-julhCJ9dXwkOg9svuuYqqjXLhVaUgyUvO2hWbTxwjvLXX2rG3VtAaB0SzxMnGTuoCZizBT7Xqqm2V7+ggrfCXA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/propagator-b3@2.4.0': resolution: {integrity: sha512-6VPsFiMUkJBre/86F0d+PZMaUCcuLA9DtZuC46KH8EeVEKZPEM2WlX35M/qmde8UpzoQL9qzdz54YjUYABt8Uw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -4003,18 +4040,36 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/resources@2.5.0': + resolution: {integrity: sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-logs@0.210.0': resolution: {integrity: sha512-YuaL92Dpyk/Kc1o4e9XiaWWwiC0aBFN+4oy+6A9TP4UNJmRymPMEX10r6EMMFMD7V0hktiSig9cwWo59peeLCQ==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.4.0 <1.10.0' + '@opentelemetry/sdk-logs@0.211.0': + resolution: {integrity: sha512-O5nPwzgg2JHzo59kpQTPUOTzFi0Nv5LxryG27QoXBciX3zWM3z83g+SNOHhiQVYRWFSxoWn1JM2TGD5iNjOwdA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + '@opentelemetry/sdk-metrics@2.4.0': resolution: {integrity: sha512-qSbfq9mXbLMqmPEjijl32f3ZEmiHekebRggPdPjhHI6t1CsAQOR2Aw/SuTDftk3/l2aaPHpwP3xM2DkgBA1ANw==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': '>=1.9.0 <1.10.0' + '@opentelemetry/sdk-metrics@2.5.0': + resolution: {integrity: sha512-BeJLtU+f5Gf905cJX9vXFQorAr6TAfK3SPvTFqP+scfIpDQEJfRaGJWta7sJgP+m4dNtBf9y3yvBKVAZZtJQVA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + '@opentelemetry/sdk-node@0.210.0': resolution: {integrity: sha512-KymqUtYvfpblDNgGxBXYqCcDjYXwjOF7Muc6ocs0rMlG/66Hcs9KiJ7hg4zLOv63JubF/vxi5WXaLrQrPKyaZQ==} engines: {node: ^18.19.0 || >=20.6.0} @@ -4027,6 +4082,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-base@2.5.0': + resolution: {integrity: sha512-VzRf8LzotASEyNDUxTdaJ9IRJ1/h692WyArDBInf5puLCjxbICD6XkHgpuudis56EndyS7LYFmtTMny6UABNdQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-node@2.4.0': resolution: {integrity: sha512-MBc2l04hZPYygnWPT38UiOPy9ueutPqmJ47z0m9IKuoVQh3MblmbSgwspjhdHagZLfSfmlzhWR1xtbgVNmjX2A==} engines: {node: ^18.19.0 || >=20.6.0} @@ -4037,12 +4098,6 @@ packages: resolution: {integrity: sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==} engines: {node: '>=14'} - '@opentelemetry/sql-common@0.41.2': - resolution: {integrity: sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==} - engines: {node: ^18.19.0 || >=20.6.0} - peerDependencies: - '@opentelemetry/api': ^1.1.0 - '@paralleldrive/cuid2@2.3.1': resolution: {integrity: sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==} @@ -5403,12 +5458,6 @@ packages: '@types/parse5@5.0.3': resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} - '@types/pg-pool@2.0.7': - resolution: {integrity: sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng==} - - '@types/pg@8.15.6': - resolution: {integrity: sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==} - '@types/pg@8.16.0': resolution: {integrity: sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==} @@ -6130,6 +6179,9 @@ packages: resolution: {integrity: sha512-bkXY9WsVpY7CvMhKSR6pZilZu9Ln5WDrKVBUXf2S443etkmEO4V58heTecXcUIsNsi4Rx8JUO4NfX1IcQl4deg==} engines: {node: '>=18.20'} + bullmq-otel@1.1.0: + resolution: {integrity: sha512-8tHl/NkZghp+YLSVvqBtB1Gox/hilNFVG4Or2NmBJQilxbC+nhfgl4+lG5QB0C3bEfLwAvBQjyDNAnGe0i5vHA==} + bullmq@5.66.5: resolution: {integrity: sha512-DC1E7P03L+TfNHv+2SGxwNYvtb0oJPODWSKkWdfis0heU5zFW16vjM7fCjwlxMdGWw2w28EI3mTRfYLEHeQQSw==} @@ -15463,6 +15515,11 @@ snapshots: '@esbuild/win32-x64@0.27.2': optional: true + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@1.21.7))': + dependencies: + eslint: 9.39.2(jiti@1.21.7) + eslint-visitor-keys: 3.4.3 + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@2.6.1))': dependencies: eslint: 9.39.2(jiti@2.6.1) @@ -16529,6 +16586,10 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs@0.211.0': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api@1.9.0': {} '@opentelemetry/configuration@0.210.0(@opentelemetry/api@1.9.0)': @@ -16546,6 +16607,11 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/core@2.5.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/exporter-logs-otlp-grpc@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.14.3 @@ -16597,6 +16663,15 @@ snapshots: '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-http@0.211.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-metrics-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16634,6 +16709,15 @@ snapshots: '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http@0.211.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-proto@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16683,18 +16767,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/instrumentation-pg@0.62.0(@opentelemetry/api@1.9.0)': - dependencies: - '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@opentelemetry/instrumentation': 0.210.0(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.38.0 - '@opentelemetry/sql-common': 0.41.2(@opentelemetry/api@1.9.0) - '@types/pg': 8.15.6 - '@types/pg-pool': 2.0.7 - transitivePeerDependencies: - - supports-color - '@opentelemetry/instrumentation@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16710,6 +16782,12 @@ snapshots: '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/otlp-transformer': 0.210.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base@0.211.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@grpc/grpc-js': 1.14.3 @@ -16729,6 +16807,17 @@ snapshots: '@opentelemetry/sdk-trace-base': 2.4.0(@opentelemetry/api@1.9.0) protobufjs: 8.0.0 + '@opentelemetry/otlp-transformer@0.211.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.211.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.211.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.5.0(@opentelemetry/api@1.9.0) + protobufjs: 8.0.0 + '@opentelemetry/propagator-b3@2.4.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16747,6 +16836,12 @@ snapshots: '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/resources@2.5.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/sdk-logs@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16754,12 +16849,25 @@ snapshots: '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs@0.211.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.211.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics@2.4.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics@2.5.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-node@0.210.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16797,6 +16905,13 @@ snapshots: '@opentelemetry/resources': 2.4.0(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/sdk-trace-base@2.5.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.5.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.38.0 + '@opentelemetry/sdk-trace-node@2.4.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -16806,11 +16921,6 @@ snapshots: '@opentelemetry/semantic-conventions@1.38.0': {} - '@opentelemetry/sql-common@0.41.2(@opentelemetry/api@1.9.0)': - dependencies: - '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 2.4.0(@opentelemetry/api@1.9.0) - '@paralleldrive/cuid2@2.3.1': dependencies: '@noble/hashes': 1.8.0 @@ -18256,16 +18366,6 @@ snapshots: '@types/parse5@5.0.3': {} - '@types/pg-pool@2.0.7': - dependencies: - '@types/pg': 8.16.0 - - '@types/pg@8.15.6': - dependencies: - '@types/node': 24.10.9 - pg-protocol: 1.11.0 - pg-types: 2.2.0 - '@types/pg@8.16.0': dependencies: '@types/node': 24.10.9 @@ -18409,6 +18509,22 @@ snapshots: dependencies: '@types/yargs-parser': 21.0.3 + '@typescript-eslint/eslint-plugin@8.53.0(@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.53.0 + '@typescript-eslint/type-utils': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/utils': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.53.0 + eslint: 9.39.2(jiti@1.21.7) + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/eslint-plugin@8.53.0(@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -18425,6 +18541,18 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.53.0 + '@typescript-eslint/types': 8.53.0 + '@typescript-eslint/typescript-estree': 8.53.0(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.53.0 + debug: 4.4.3 + eslint: 9.39.2(jiti@1.21.7) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/scope-manager': 8.53.0 @@ -18455,6 +18583,18 @@ snapshots: dependencies: typescript: 5.9.3 + '@typescript-eslint/type-utils@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.53.0 + '@typescript-eslint/typescript-estree': 8.53.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + debug: 4.4.3 + eslint: 9.39.2(jiti@1.21.7) + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/type-utils@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.53.0 @@ -18484,6 +18624,17 @@ snapshots: transitivePeerDependencies: - supports-color + '@typescript-eslint/utils@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@1.21.7)) + '@typescript-eslint/scope-manager': 8.53.0 + '@typescript-eslint/types': 8.53.0 + '@typescript-eslint/typescript-estree': 8.53.0(typescript@5.9.3) + eslint: 9.39.2(jiti@1.21.7) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + '@typescript-eslint/utils@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) @@ -18504,6 +18655,25 @@ snapshots: '@vercel/oidc@3.0.5': {} + '@vitest/coverage-v8@3.2.4(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@1.21.7)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@ampproject/remapping': 2.3.0 + '@bcoe/v8-coverage': 1.0.2 + ast-v8-to-istanbul: 0.3.8 + debug: 4.4.3 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-report: 3.0.1 + istanbul-lib-source-maps: 5.0.6 + istanbul-reports: 3.2.0 + magic-string: 0.30.21 + magicast: 0.3.5 + std-env: 3.10.0 + test-exclude: 7.0.1 + tinyrainbow: 2.0.0 + vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@1.21.7)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + transitivePeerDependencies: + - supports-color + '@vitest/coverage-v8@3.2.4(vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@2.6.1)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@ampproject/remapping': 2.3.0 @@ -18550,6 +18720,14 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 3.2.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@24.10.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 3.2.4 @@ -19163,6 +19341,13 @@ snapshots: builtin-modules@5.0.0: {} + bullmq-otel@1.1.0: + dependencies: + '@opentelemetry/api': 1.9.0 + bullmq: 5.66.5 + transitivePeerDependencies: + - supports-color + bullmq@5.66.5: dependencies: cron-parser: 4.9.0 @@ -20552,6 +20737,10 @@ snapshots: source-map: 0.6.1 optional: true + eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@1.21.7)): + dependencies: + eslint: 9.39.2(jiti@1.21.7) + eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@2.6.1)): dependencies: eslint: 9.39.2(jiti@2.6.1) @@ -20568,6 +20757,16 @@ snapshots: lodash.memoize: 4.1.2 semver: 7.7.3 + eslint-plugin-prettier@5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@1.21.7)))(eslint@9.39.2(jiti@1.21.7))(prettier@3.8.0): + dependencies: + eslint: 9.39.2(jiti@1.21.7) + prettier: 3.8.0 + prettier-linter-helpers: 1.0.1 + synckit: 0.11.12 + optionalDependencies: + '@types/eslint': 9.6.1 + eslint-config-prettier: 10.1.8(eslint@9.39.2(jiti@1.21.7)) + eslint-plugin-prettier@5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@10.1.8(eslint@9.39.2(jiti@2.6.1)))(eslint@9.39.2(jiti@2.6.1))(prettier@3.8.0): dependencies: eslint: 9.39.2(jiti@2.6.1) @@ -20596,6 +20795,28 @@ snapshots: transitivePeerDependencies: - ts-node + eslint-plugin-unicorn@62.0.0(eslint@9.39.2(jiti@1.21.7)): + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@1.21.7)) + '@eslint/plugin-kit': 0.4.1 + change-case: 5.4.4 + ci-info: 4.3.1 + clean-regexp: 1.0.0 + core-js-compat: 3.47.0 + eslint: 9.39.2(jiti@1.21.7) + esquery: 1.6.0 + find-up-simple: 1.0.1 + globals: 16.5.0 + indent-string: 5.0.0 + is-builtin-module: 5.0.0 + jsesc: 3.1.0 + pluralize: 8.0.0 + regexp-tree: 0.1.27 + regjsparser: 0.13.0 + semver: 7.7.3 + strip-indent: 4.1.1 + eslint-plugin-unicorn@62.0.0(eslint@9.39.2(jiti@2.6.1)): dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -20632,6 +20853,47 @@ snapshots: eslint-visitor-keys@4.2.1: {} + eslint@9.39.2(jiti@1.21.7): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@1.21.7)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.1 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.3 + '@eslint/js': 9.39.2 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.6.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 1.21.7 + transitivePeerDependencies: + - supports-color + eslint@9.39.2(jiti@2.6.1): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) @@ -26178,6 +26440,17 @@ snapshots: typedarray@0.0.6: {} + typescript-eslint@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.53.0(@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/parser': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.53.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.53.0(eslint@9.39.2(jiti@1.21.7))(typescript@5.9.3) + eslint: 9.39.2(jiti@1.21.7) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + typescript-eslint@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): dependencies: '@typescript-eslint/eslint-plugin': 8.53.0(@typescript-eslint/parser@8.53.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) @@ -26463,6 +26736,27 @@ snapshots: - rollup - supports-color + vite-node@3.2.4(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + vite-node@3.2.4(@types/node@24.10.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: cac: 6.7.14 @@ -26505,6 +26799,17 @@ snapshots: - tsx - yaml + vite-tsconfig-paths@6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)): + dependencies: + debug: 4.4.3 + globrex: 0.1.2 + tsconfck: 3.1.6(typescript@5.9.3) + optionalDependencies: + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + transitivePeerDependencies: + - supports-color + - typescript + vite-tsconfig-paths@6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@24.10.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)): dependencies: debug: 4.4.3 @@ -26516,6 +26821,24 @@ snapshots: - supports-color - typescript + vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): + dependencies: + esbuild: 0.27.2 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.55.1 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 24.10.9 + fsevents: 2.3.3 + jiti: 1.21.7 + lightningcss: 1.30.2 + sass: 1.97.1 + terser: 5.44.1 + tsx: 4.21.0 + yaml: 2.8.2 + vite@7.3.1(@types/node@24.10.9)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: esbuild: 0.27.2 @@ -26560,6 +26883,50 @@ snapshots: dependencies: vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@2.6.1)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@1.21.7)(jsdom@26.1.0(canvas@2.11.2))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.4 + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/pretty-format': 3.2.4 + '@vitest/runner': 3.2.4 + '@vitest/snapshot': 3.2.4 + '@vitest/spy': 3.2.4 + '@vitest/utils': 3.2.4 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.3.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + vite-node: 3.2.4(@types/node@24.10.9)(jiti@1.21.7)(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/debug': 4.1.12 + '@types/node': 24.10.9 + happy-dom: 20.3.0 + jsdom: 26.1.0(canvas@2.11.2) + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(happy-dom@20.3.0)(jiti@2.6.1)(jsdom@26.1.0(canvas@2.11.2(encoding@0.1.13)))(lightningcss@1.30.2)(sass@1.97.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 diff --git a/server/bin/immich-dev b/server/bin/immich-dev index 84c5eea8da..47ef95e7cb 100755 --- a/server/bin/immich-dev +++ b/server/bin/immich-dev @@ -6,4 +6,17 @@ if [[ "$IMMICH_ENV" == "production" ]]; then fi cd /usr/src/app || exit + +# Compile telemetry-preload so it can be required before the app starts. +# This is necessary for OpenTelemetry http instrumentation to work correctly. +mkdir -p dist +pnpm --filter immich exec tsc src/telemetry-preload.ts \ + --outDir dist \ + --esModuleInterop \ + --module node16 \ + --moduleResolution node16 \ + --target ES2022 \ + --skipLibCheck + +export NODE_OPTIONS="${NODE_OPTIONS} --require ./dist/telemetry-preload.js" pnpm --filter immich exec nest start --debug "0.0.0.0:9230" --watch -- "$@" diff --git a/server/bin/start.sh b/server/bin/start.sh index 0afff4c3a8..e4be511a54 100755 --- a/server/bin/start.sh +++ b/server/bin/start.sh @@ -39,7 +39,7 @@ else fi if [ -f "${SERVER_HOME}/dist/main.js" ]; then - exec node "${SERVER_HOME}/dist/main.js" "$@" + exec node --require "${SERVER_HOME}/dist/telemetry-preload.js" "${SERVER_HOME}/dist/main.js" "$@" else echo "Error: ${SERVER_HOME}/dist/main.js not found" if [ "$IMMICH_ENV" = "development" ]; then diff --git a/server/package.json b/server/package.json index e84cc2f9eb..1a1b69af55 100644 --- a/server/package.json +++ b/server/package.json @@ -45,14 +45,16 @@ "@nestjs/websockets": "^11.0.4", "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "^2.0.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.211.0", "@opentelemetry/exporter-prometheus": "^0.210.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.211.0", "@opentelemetry/instrumentation-http": "^0.210.0", "@opentelemetry/instrumentation-ioredis": "^0.58.0", "@opentelemetry/instrumentation-nestjs-core": "^0.56.0", - "@opentelemetry/instrumentation-pg": "^0.62.0", "@opentelemetry/resources": "^2.0.1", "@opentelemetry/sdk-metrics": "^2.0.1", "@opentelemetry/sdk-node": "^0.210.0", + "@opentelemetry/sdk-trace-base": "^2.5.0", "@opentelemetry/semantic-conventions": "^1.34.0", "@react-email/components": "^0.5.0", "@react-email/render": "^1.1.2", @@ -63,6 +65,7 @@ "bcrypt": "^6.0.0", "body-parser": "^2.2.0", "bullmq": "^5.51.0", + "bullmq-otel": "^1.1.0", "chokidar": "^4.0.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", diff --git a/server/src/enum.ts b/server/src/enum.ts index 8f509754da..57c34d2df0 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -47,6 +47,13 @@ export enum AssetFileType { Sidecar = 'sidecar', } +export enum GeneratedImageType { + Thumbnail = 'thumbnail', + Preview = 'preview', + Fullsize = 'fullsize', + Person = 'person', +} + export enum AlbumUserRole { Editor = 'editor', Viewer = 'viewer', diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 54a5d1987f..87e49e38cb 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -8,6 +8,7 @@ import { RedisOptions } from 'ioredis'; import { CLS_ID, ClsModuleOptions } from 'nestjs-cls'; import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { join } from 'node:path'; +import { bullMQTelemetry } from 'src/telemetry-preload'; import { citiesFile, excludePaths, IWorker } from 'src/constants'; import { Telemetry } from 'src/decorators'; import { EnvDto } from 'src/dtos/env.dto'; @@ -262,6 +263,7 @@ const getEnv = (): EnvData => { removeOnComplete: true, removeOnFail: false, }, + telemetry: bullMQTelemetry, }, queues: Object.values(QueueName).map((name) => ({ name })), }, diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.ts index 49778b5193..63876104af 100644 --- a/server/src/repositories/machine-learning.repository.ts +++ b/server/src/repositories/machine-learning.repository.ts @@ -1,9 +1,13 @@ import { Injectable } from '@nestjs/common'; +import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; import { Duration } from 'luxon'; import { readFile } from 'node:fs/promises'; import { MachineLearningConfig } from 'src/config'; import { CLIPConfig } from 'src/dtos/model-config.dto'; import { LoggingRepository } from 'src/repositories/logging.repository'; +import { Traced } from 'src/utils/instrumentation'; + +const tracer = trace.getTracer('immich'); export interface BoundingBox { x1: number; @@ -162,36 +166,60 @@ export class MachineLearningRepository { } private async predict(payload: ModelPayload, config: MachineLearningRequest): Promise { - const formData = await this.getFormData(payload, config); + const taskType = Object.keys(config)[0] as ModelTask; - for (const url of [ - // try healthy servers first - ...this.config.urls.filter((url) => this.isHealthy(url)), - ...this.config.urls.filter((url) => !this.isHealthy(url)), - ]) { - try { - const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData }); - if (response.ok) { - this.setHealthy(url, true); - return response.json(); + return tracer.startActiveSpan('ml.predict', { kind: SpanKind.CLIENT }, context.active(), async (span: Span) => { + span.setAttribute('ml.task', taskType); + + const formData = await this.getFormData(payload, config); + let attemptCount = 0; + let successUrl: string | undefined; + + const urls = [ + ...this.config.urls.filter((url) => this.isHealthy(url)), + ...this.config.urls.filter((url) => !this.isHealthy(url)), + ]; + + for (const url of urls) { + attemptCount++; + try { + const response = await fetch(new URL('/predict', url), { method: 'POST', body: formData }); + if (response.ok) { + this.setHealthy(url, true); + successUrl = url; + span.setAttribute('ml.url', url); + span.setAttribute('ml.attempts', attemptCount); + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return response.json(); + } + + this.logger.warn( + `Machine learning request to "${url}" failed with status ${response.status}: ${response.statusText}`, + ); + } catch (error: Error | unknown) { + this.logger.warn( + `Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`, + ); } - this.logger.warn( - `Machine learning request to "${url}" failed with status ${response.status}: ${response.statusText}`, - ); - } catch (error: Error | unknown) { - this.logger.warn( - `Machine learning request to "${url}" failed: ${error instanceof Error ? error.message : error}`, - ); + this.setHealthy(url, false); } - this.setHealthy(url, false); - } - - throw new Error(`Machine learning request '${JSON.stringify(config)}' failed for all URLs`); + span.setAttribute('ml.attempts', attemptCount); + const errorMessage = `Machine learning request '${JSON.stringify(config)}' failed for all URLs`; + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + span.end(); + throw new Error(errorMessage); + }); } + @Traced({ name: 'ml.detectFaces', kind: SpanKind.CLIENT }) async detectFaces(imagePath: string, { modelName, minScore }: FaceDetectionOptions) { + const span = trace.getActiveSpan(); + span?.setAttribute('ml.modelName', modelName); + span?.setAttribute('ml.minScore', minScore); + const request = { [ModelTask.FACIAL_RECOGNITION]: { [ModelType.DETECTION]: { modelName, options: { minScore } }, @@ -206,19 +234,36 @@ export class MachineLearningRepository { }; } + @Traced({ name: 'ml.encodeImage', kind: SpanKind.CLIENT }) async encodeImage(imagePath: string, { modelName }: CLIPConfig) { + trace.getActiveSpan()?.setAttribute('ml.modelName', modelName); + const request = { [ModelTask.SEARCH]: { [ModelType.VISUAL]: { modelName } } }; const response = await this.predict({ imagePath }, request); return response[ModelTask.SEARCH]; } + @Traced({ name: 'ml.encodeText', kind: SpanKind.CLIENT }) async encodeText(text: string, { language, modelName }: TextEncodingOptions) { + const span = trace.getActiveSpan(); + span?.setAttribute('ml.modelName', modelName); + if (language) { + span?.setAttribute('ml.language', language); + } + const request = { [ModelTask.SEARCH]: { [ModelType.TEXTUAL]: { modelName, options: { language } } } }; const response = await this.predict({ text }, request); return response[ModelTask.SEARCH]; } + @Traced({ name: 'ml.ocr', kind: SpanKind.CLIENT }) async ocr(imagePath: string, { modelName, minDetectionScore, minRecognitionScore, maxResolution }: OcrOptions) { + const span = trace.getActiveSpan(); + span?.setAttribute('ml.modelName', modelName); + span?.setAttribute('ml.minDetectionScore', minDetectionScore); + span?.setAttribute('ml.minRecognitionScore', minRecognitionScore); + span?.setAttribute('ml.maxResolution', maxResolution); + const request = { [ModelTask.OCR]: { [ModelType.DETECTION]: { modelName, options: { minScore: minDetectionScore, maxResolution } }, diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 33025e73cf..cd7935acc3 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import { Duration } from 'luxon'; @@ -19,6 +20,7 @@ import { TranscodeCommand, VideoInfo, } from 'src/types'; +import { Traced } from 'src/utils/instrumentation'; import { handlePromiseError } from 'src/utils/misc'; import { createAffineMatrix } from 'src/utils/transform'; @@ -26,9 +28,17 @@ const probe = (input: string, options: string[]): Promise => new Promise((resolve, reject) => ffmpeg.ffprobe(input, options, (error, data) => (error ? reject(error) : resolve(data))), ); +const tracer = trace.getTracer('immich'); sharp.concurrency(0); sharp.cache({ files: 0 }); +const EXTRACTION_METHODS = [ + { tag: 'JpgFromRaw2', format: RawExtractedFormat.Jpeg }, + { tag: 'JpgFromRaw', format: RawExtractedFormat.Jpeg }, + { tag: 'PreviewJXL', format: RawExtractedFormat.Jxl }, + { tag: 'PreviewImage', format: RawExtractedFormat.Jpeg }, +] as const; + type ProgressEvent = { frames: number; currentFps: number; @@ -49,81 +59,66 @@ export class MediaRepository { this.logger.setContext(MediaRepository.name); } - /** - * - * @param input file path to the input image - * @returns ExtractResult if succeeded, or null if failed - */ + @Traced('media.exiftool.extract') async extract(input: string): Promise { - try { - const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw2', input); - return { buffer, format: RawExtractedFormat.Jpeg }; - } catch (error: any) { - this.logger.debug(`Could not extract JpgFromRaw2 buffer from image, trying JPEG from RAW next: ${error}`); - } - - try { - const buffer = await exiftool.extractBinaryTagToBuffer('JpgFromRaw', input); - return { buffer, format: RawExtractedFormat.Jpeg }; - } catch (error: any) { - this.logger.debug(`Could not extract JPEG buffer from image, trying PreviewJXL next: ${error}`); - } - - try { - const buffer = await exiftool.extractBinaryTagToBuffer('PreviewJXL', input); - return { buffer, format: RawExtractedFormat.Jxl }; - } catch (error: any) { - this.logger.debug(`Could not extract PreviewJXL buffer from image, trying PreviewImage next: ${error}`); - } - - try { - const buffer = await exiftool.extractBinaryTagToBuffer('PreviewImage', input); - return { buffer, format: RawExtractedFormat.Jpeg }; - } catch (error: any) { - this.logger.debug(`Could not extract preview buffer from image: ${error}`); - return null; + for (const { tag, format } of EXTRACTION_METHODS) { + try { + const buffer = await exiftool.extractBinaryTagToBuffer(tag, input); + trace.getActiveSpan()?.setAttribute('exiftool.extractedTag', tag); + return { buffer, format }; + } catch { + this.logger.debug(`Could not extract ${tag} from image, trying next method`); + } } + trace.getActiveSpan()?.setAttribute('exiftool.extractedTag', 'none'); + return null; } + @Traced('media.exiftool.writeExif') async writeExif(tags: Partial, output: string): Promise { - try { - const tagsToWrite: WriteTags = { - ExifImageWidth: tags.exifImageWidth, - ExifImageHeight: tags.exifImageHeight, - DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()), - ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()), - TimeZone: tags.timeZone, - GPSLatitude: tags.latitude, - GPSLongitude: tags.longitude, - ProjectionType: tags.projectionType, - City: tags.city, - Country: tags.country, - Make: tags.make, - Model: tags.model, - LensModel: tags.lensModel, - Fnumber: tags.fNumber?.toFixed(1), - FocalLength: tags.focalLength?.toFixed(1), - ISO: tags.iso, - ExposureTime: tags.exposureTime, - ProfileDescription: tags.profileDescription, - ColorSpace: tags.colorspace, - Rating: tags.rating, - // specially convert Orientation to numeric Orientation# for exiftool - 'Orientation#': tags.orientation ? Number(tags.orientation) : undefined, - }; + const tagsToWrite: WriteTags = { + ExifImageWidth: tags.exifImageWidth, + ExifImageHeight: tags.exifImageHeight, + DateTimeOriginal: tags.dateTimeOriginal && ExifDateTime.fromMillis(tags.dateTimeOriginal.getTime()), + ModifyDate: tags.modifyDate && ExifDateTime.fromMillis(tags.modifyDate.getTime()), + TimeZone: tags.timeZone, + GPSLatitude: tags.latitude, + GPSLongitude: tags.longitude, + ProjectionType: tags.projectionType, + City: tags.city, + Country: tags.country, + Make: tags.make, + Model: tags.model, + LensModel: tags.lensModel, + Fnumber: tags.fNumber?.toFixed(1), + FocalLength: tags.focalLength?.toFixed(1), + ISO: tags.iso, + ExposureTime: tags.exposureTime, + ProfileDescription: tags.profileDescription, + ColorSpace: tags.colorspace, + Rating: tags.rating, + 'Orientation#': tags.orientation ? Number(tags.orientation) : undefined, + }; + try { await exiftool.write(output, tagsToWrite, { ignoreMinorErrors: true, writeArgs: ['-overwrite_original'], }); return true; - } catch (error: any) { - this.logger.warn(`Could not write exif data to image: ${error.message}`); + } catch (error: unknown) { + const span = trace.getActiveSpan(); + span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span?.recordException(error as Error); + this.logger.warn(`Could not write exif data to image: ${error}`); return false; } } + @Traced('media.exiftool.copyTagGroup') async copyTagGroup(tagGroup: string, source: string, target: string): Promise { + trace.getActiveSpan()?.setAttribute('exiftool.tagGroup', tagGroup); + try { await exiftool.write( target, @@ -134,15 +129,28 @@ export class MediaRepository { }, ); return true; - } catch (error: any) { - this.logger.warn(`Could not copy tag data to image: ${error.message}`); + } catch (error: unknown) { + const span = trace.getActiveSpan(); + span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span?.recordException(error as Error); + this.logger.warn(`Could not copy tag data to image: ${error}`); return false; } } + @Traced('media.decodeImage') async decodeImage(input: string | Buffer, options: DecodeToBufferOptions) { + const span = trace.getActiveSpan(); + span?.setAttribute('media.colorspace', options.colorspace); + if (options.size !== undefined) { + span?.setAttribute('media.size', options.size); + } + const pipeline = await this.getImageDecodingPipeline(input, options); - return pipeline.raw().toBuffer({ resolveWithObject: true }); + const result = await pipeline.raw().toBuffer({ resolveWithObject: true }); + span?.setAttribute('media.output.width', result.info.width); + span?.setAttribute('media.output.height', result.info.height); + return result; } private async applyEdits(pipeline: sharp.Sharp, edits: AssetEditActionItem[]): Promise { @@ -170,11 +178,21 @@ export class MediaRepository { return pipeline; } + @Traced('media.generateThumbnail') async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise { + const span = trace.getActiveSpan(); + span?.setAttribute('media.imageType', options.imageType ?? 'unknown'); + span?.setAttribute('media.format', options.format); + span?.setAttribute('media.quality', options.quality); + span?.setAttribute('media.size', options.size ?? 0); + span?.setAttribute('media.colorspace', options.colorspace); + if (options.progressive !== undefined) { + span?.setAttribute('media.progressive', options.progressive); + } + const pipeline = await this.getImageDecodingPipeline(input, options); const decoded = pipeline.toFormat(options.format, { quality: options.quality, - // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0', progressive: options.progressive, }); @@ -215,7 +233,11 @@ export class MediaRepository { return pipeline; } + @Traced('media.generateThumbhash') async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise { + const span = trace.getActiveSpan(); + span?.setAttribute('media.colorspace', options.colorspace); + const [{ rgbaToThumbHash }, decodingPipeline] = await Promise.all([ import('thumbhash'), this.getImageDecodingPipeline(input, { @@ -229,12 +251,23 @@ export class MediaRepository { const pipeline = decodingPipeline.resize(100, 100, { fit: 'inside', withoutEnlargement: true }).raw().ensureAlpha(); const { data, info } = await pipeline.toBuffer({ resolveWithObject: true }); + span?.setAttribute('media.thumbhash.width', info.width); + span?.setAttribute('media.thumbhash.height', info.height); return Buffer.from(rgbaToThumbHash(info.width, info.height, data)); } + @Traced('ffmpeg.probe') async probe(input: string, options?: ProbeOptions): Promise { + const span = trace.getActiveSpan(); + span?.setAttribute('ffmpeg.countFrames', options?.countFrames ?? false); + const results = await probe(input, options?.countFrames ? ['-count_packets'] : []); // gets frame count quickly: https://stackoverflow.com/a/28376817 + + span?.setAttribute('ffmpeg.duration', this.parseFloat(results.format.duration)); + span?.setAttribute('ffmpeg.videoStreams', results.streams.filter((s) => s.codec_type === 'video').length); + span?.setAttribute('ffmpeg.audioStreams', results.streams.filter((s) => s.codec_type === 'audio').length); + return { format: { formatName: results.format.format_name, @@ -272,17 +305,39 @@ export class MediaRepository { } transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise { + const span = tracer.startSpan('ffmpeg.transcode', { kind: SpanKind.INTERNAL }, context.active()); + span.setAttribute('ffmpeg.twoPass', options.twoPass ?? false); + span.setAttribute('ffmpeg.frameCount', options.progress.frameCount); + + const endSpan = (error?: Error) => { + if (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + span.recordException(error); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + span.end(); + }; + if (!options.twoPass) { return new Promise((resolve, reject) => { this.configureFfmpegCall(input, output, options) - .on('error', reject) - .on('end', () => resolve()) + .on('error', (error) => { + endSpan(error); + reject(error); + }) + .on('end', () => { + endSpan(); + resolve(); + }) .run(); }); } if (typeof output !== 'string') { - throw new TypeError('Two-pass transcoding does not support writing to a stream'); + const error = new TypeError('Two-pass transcoding does not support writing to a stream'); + endSpan(error); + throw error; } // two-pass allows for precise control of bitrate at the cost of running twice @@ -293,24 +348,37 @@ export class MediaRepository { .addOptions('-pass', '1') .addOptions('-passlogfile', output) .addOptions('-f null') - .on('error', reject) + .on('error', (error) => { + endSpan(error); + reject(error); + }) .on('end', () => { // second pass this.configureFfmpegCall(input, output, options) .addOptions('-pass', '2') .addOptions('-passlogfile', output) - .on('error', reject) + .on('error', (error) => { + endSpan(error); + reject(error); + }) .on('end', () => handlePromiseError(fs.unlink(`${output}-0.log`), this.logger)) .on('end', () => handlePromiseError(fs.rm(`${output}-0.log.mbtree`, { force: true }), this.logger)) - .on('end', () => resolve()) + .on('end', () => { + endSpan(); + resolve(); + }) .run(); }) .run(); }); } + @Traced('media.getImageDimensions') async getImageDimensions(input: string | Buffer): Promise { const { width = 0, height = 0 } = await sharp(input).metadata(); + const span = trace.getActiveSpan(); + span?.setAttribute('media.width', width); + span?.setAttribute('media.height', height); return { width, height }; } diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts index 1334d1220f..37d81e810a 100644 --- a/server/src/repositories/metadata.repository.ts +++ b/server/src/repositories/metadata.repository.ts @@ -1,7 +1,9 @@ import { Injectable } from '@nestjs/common'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; import { BinaryField, DefaultReadTaskOptions, ExifTool, Tags } from 'exiftool-vendored'; import geotz from 'geo-tz'; import { LoggingRepository } from 'src/repositories/logging.repository'; +import { Traced } from 'src/utils/instrumentation'; import { mimeTypes } from 'src/utils/mime-types'; interface ExifDuration { @@ -104,22 +106,34 @@ export class MetadataRepository { await this.exiftool.end(); } - readTags(path: string): Promise { - const args = mimeTypes.isVideo(path) ? ['-ee'] : []; - return this.exiftool.read(path, args).catch((error) => { - this.logger.warn(`Error reading exif data (${path}): ${error}\n${error?.stack}`); + @Traced('metadata.exiftool.readTags') + async readTags(path: string): Promise { + try { + const args = mimeTypes.isVideo(path) ? ['-ee'] : []; + return (await this.exiftool.read(path, args)) as ImmichTags; + } catch (error: unknown) { + const span = trace.getActiveSpan(); + span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span?.recordException(error as Error); + this.logger.warn(`Error reading exif data (${path}): ${error}`); return {}; - }) as Promise; + } } - extractBinaryTag(path: string, tagName: string): Promise { + @Traced('metadata.exiftool.extractBinaryTag') + async extractBinaryTag(path: string, tagName: string): Promise { + trace.getActiveSpan()?.setAttribute('exiftool.tagName', tagName); return this.exiftool.extractBinaryTagToBuffer(tagName, path); } + @Traced('metadata.exiftool.writeTags') async writeTags(path: string, tags: Partial): Promise { try { await this.exiftool.write(path, tags); - } catch (error) { + } catch (error: unknown) { + const span = trace.getActiveSpan(); + span?.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span?.recordException(error as Error); this.logger.warn(`Error writing exif data (${path}): ${error}`); } } diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts index 7345dfef5b..542d95f0a3 100644 --- a/server/src/repositories/storage.repository.ts +++ b/server/src/repositories/storage.repository.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { trace } from '@opentelemetry/api'; import archiver from 'archiver'; import chokidar, { ChokidarOptions } from 'chokidar'; import { escapePath, glob, globStream } from 'fast-glob'; @@ -9,6 +10,7 @@ import { PassThrough, Readable, Writable } from 'node:stream'; import { createGunzip, createGzip } from 'node:zlib'; import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto'; import { LoggingRepository } from 'src/repositories/logging.repository'; +import { Traced } from 'src/utils/instrumentation'; import { mimeTypes } from 'src/utils/mime-types'; export interface WatchEvents { @@ -106,9 +108,18 @@ export class StorageRepository { return createReadStream(filepath); } + @Traced('storage.createReadStream') async createReadStream(filepath: string, mimeType?: string | null): Promise { const { size } = await fs.stat(filepath); await fs.access(filepath, constants.R_OK); + + const span = trace.getActiveSpan(); + span?.setAttribute('file.path', filepath); + span?.setAttribute('file.size', size); + if (mimeType) { + span?.setAttribute('file.content_type', mimeType); + } + return { stream: createReadStream(filepath), length: size, @@ -116,10 +127,15 @@ export class StorageRepository { }; } + @Traced('storage.readFile') async readFile(filepath: string, options?: ReadOptionsWithBuffer): Promise { + const span = trace.getActiveSpan(); + span?.setAttribute('file.path', filepath); + const file = await fs.open(filepath); try { - const { buffer } = await file.read(options); + const { buffer, bytesRead } = await file.read(options); + span?.setAttribute('file.bytes_read', bytesRead); return buffer as Buffer; } finally { await file.close(); diff --git a/server/src/repositories/telemetry.repository.ts b/server/src/repositories/telemetry.repository.ts index 5fbbb76cf7..8d3e3bd21b 100644 --- a/server/src/repositories/telemetry.repository.ts +++ b/server/src/repositories/telemetry.repository.ts @@ -1,21 +1,11 @@ import { Injectable } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import { MetricOptions } from '@opentelemetry/api'; -import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; -import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; -import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; -import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; -import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; -import { resourceFromAttributes } from '@opentelemetry/resources'; -import { AggregationType } from '@opentelemetry/sdk-metrics'; -import { NodeSDK, contextBase } from '@opentelemetry/sdk-node'; -import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; +import { contextBase } from '@opentelemetry/sdk-node'; import { ClassConstructor } from 'class-transformer'; import { snakeCase, startCase } from 'lodash'; import { MetricService } from 'nestjs-otel'; import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils'; -import { serverVersion } from 'src/constants'; import { ImmichTelemetry, MetadataKey } from 'src/enum'; import { ConfigRepository } from 'src/repositories/config.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -51,49 +41,16 @@ export class MetricGroupRepository { } } -const aggregationBoundaries = [ - 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000, -]; +// OpenTelemetry SDK is now initialized in telemetry-preload.ts +// which runs before any modules are imported. This ensures the +// global meter provider is available for nestjs-otel. -let instance: NodeSDK | undefined; - -export const bootstrapTelemetry = (port: number) => { - if (instance) { - throw new Error('OpenTelemetry SDK already started'); - } - instance = new NodeSDK({ - resource: resourceFromAttributes({ - [ATTR_SERVICE_NAME]: `immich`, - [ATTR_SERVICE_VERSION]: serverVersion.toString(), - }), - metricReader: new PrometheusExporter({ port }), - contextManager: new AsyncLocalStorageContextManager(), - instrumentations: [ - new HttpInstrumentation(), - new IORedisInstrumentation(), - new NestInstrumentation(), - new PgInstrumentation(), - ], - views: [ - { - instrumentName: '*', - instrumentUnit: 'ms', - aggregation: { - type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, - options: { boundaries: aggregationBoundaries }, - }, - }, - ], - }); - - instance.start(); +export const bootstrapTelemetry = (_port: number) => { + // No-op: SDK is initialized in telemetry-preload.ts }; export const teardownTelemetry = async () => { - if (instance) { - await instance.shutdown(); - instance = undefined; - } + // No-op: SDK shutdown is handled by process exit }; @Injectable() diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 8e6440eb7a..73e42c88d8 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -8,6 +8,7 @@ import { AudioCodec, Colorspace, ExifOrientation, + GeneratedImageType, ImageFormat, JobName, JobStatus, @@ -356,6 +357,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, expect.any(String), ); @@ -370,6 +372,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Thumbnail, }, expect.any(String), ); @@ -581,6 +584,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, previewPath, ); @@ -595,6 +599,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Thumbnail, }, thumbnailPath, ); @@ -630,6 +635,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, previewPath, ); @@ -644,6 +650,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Thumbnail, }, thumbnailPath, ); @@ -838,6 +845,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, expect.any(String), ); @@ -870,6 +878,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Fullsize, }, expect.any(String), ); @@ -884,6 +893,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, expect.any(String), ); @@ -914,6 +924,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Fullsize, }, expect.any(String), ); @@ -928,6 +939,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Preview, }, expect.any(String), ); @@ -959,6 +971,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Fullsize, }, expect.any(String), ); @@ -1016,6 +1029,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Fullsize, }, expect.any(String), ); @@ -1056,6 +1070,7 @@ describe(MediaService.name, () => { processInvalidImages: false, raw: rawInfo, edits: [], + imageType: GeneratedImageType.Fullsize, }, expect.any(String), ); @@ -1300,6 +1315,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1345,6 +1361,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1388,6 +1405,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1431,6 +1449,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1474,6 +1493,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1517,6 +1537,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); @@ -1565,6 +1586,7 @@ describe(MediaService.name, () => { raw: info, processInvalidImages: false, size: 250, + imageType: GeneratedImageType.Person, }, expect.any(String), ); diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 8684b78c2f..8c338916c6 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -12,6 +12,7 @@ import { AssetVisibility, AudioCodec, Colorspace, + GeneratedImageType, ImageFormat, JobName, JobStatus, @@ -329,12 +330,12 @@ export class MediaService extends BaseService { this.mediaRepository.generateThumbhash(data, thumbnailOptions), this.mediaRepository.generateThumbnail( data, - { ...image.thumbnail, ...thumbnailOptions, edits: useEdits ? asset.edits : [] }, + { ...image.thumbnail, ...thumbnailOptions, imageType: GeneratedImageType.Thumbnail, edits: useEdits ? asset.edits : [] }, thumbnailPath, ), this.mediaRepository.generateThumbnail( data, - { ...image.preview, ...thumbnailOptions, edits: useEdits ? asset.edits : [] }, + { ...image.preview, ...thumbnailOptions, imageType: GeneratedImageType.Preview, edits: useEdits ? asset.edits : [] }, previewPath, ), ]; @@ -352,6 +353,7 @@ export class MediaService extends BaseService { format: image.fullsize.format, quality: image.fullsize.quality, progressive: image.fullsize.progressive, + imageType: GeneratedImageType.Fullsize, ...thumbnailOptions, }; promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath)); @@ -438,6 +440,7 @@ export class MediaService extends BaseService { progressive: false, processInvalidImages: false, size: FACE_THUMBNAIL_SIZE, + imageType: GeneratedImageType.Person, edits: [ { action: AssetEditAction.Crop, diff --git a/server/src/telemetry-preload.ts b/server/src/telemetry-preload.ts new file mode 100644 index 0000000000..5ce8aa4822 --- /dev/null +++ b/server/src/telemetry-preload.ts @@ -0,0 +1,149 @@ +import '@opentelemetry/api'; +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import type { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; +import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { AggregationType, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; +import { + ATTR_VCS_REPOSITORY_REF_REVISION, + ATTR_VCS_REPOSITORY_URL_FULL, +} from '@opentelemetry/semantic-conventions/incubating'; +import { isMainThread } from 'node:worker_threads'; +import { BullMQOtel } from 'bullmq-otel'; + +export const bullMQTelemetry = new BullMQOtel('immich-jobs'); + +const ignoredRoutes = new Set(['/api/server/ping']); +const ignoredCallbacks = new Set(['pingServer']); + +class FilteringSpanProcessor implements SpanProcessor { + constructor(private delegate: SpanProcessor) {} + + onStart(span: ReadableSpan, context: unknown): void { + (this.delegate.onStart as (span: ReadableSpan, context: unknown) => void)(span, context); + } + + onEnd(span: ReadableSpan): void { + const route = span.attributes['http.route'] || span.attributes['http.target']; + if (typeof route === 'string' && ignoredRoutes.has(route)) { + return; + } + + const callback = span.attributes['nestjs.callback']; + if (typeof callback === 'string' && ignoredCallbacks.has(callback)) { + return; + } + + this.delegate.onEnd(span); + } + + shutdown(): Promise { + return this.delegate.shutdown(); + } + + forceFlush(): Promise { + return this.delegate.forceFlush(); + } +} + +// Detect worker type to determine metrics port: +// - Forked child (API worker): process.send exists, use IMMICH_API_METRICS_PORT +// - Worker thread (microservices): !isMainThread, use IMMICH_MICROSERVICES_METRICS_PORT +// - Main process: don't start metrics (avoid port conflicts with workers) +const isForkedChild = typeof process.send === 'function'; +const isWorkerThread = !isMainThread; +const isWorkerProcess = isForkedChild || isWorkerThread; + +const telemetryInclude = process.env.IMMICH_TELEMETRY_INCLUDE; +const tracingEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; +const metricsEnabled = telemetryInclude && telemetryInclude.length > 0; + +// Only start SDK in worker processes (not main/supervisor process) +if (isWorkerProcess && (tracingEndpoint || metricsEnabled)) { + const workerType = isForkedChild ? 'api' : 'microservices'; + + const attributes: Record = { + [ATTR_SERVICE_NAME]: `immich-${workerType}`, + [ATTR_SERVICE_VERSION]: process.env.npm_package_version || 'dev', + }; + + if (process.env.IMMICH_ENV) { + attributes['deployment.environment.name'] = process.env.IMMICH_ENV; + } + + if (process.env.IMMICH_SOURCE_COMMIT) { + attributes[ATTR_VCS_REPOSITORY_REF_REVISION] = process.env.IMMICH_SOURCE_COMMIT; + } + + if (process.env.IMMICH_REPOSITORY_URL) { + attributes[ATTR_VCS_REPOSITORY_URL_FULL] = process.env.IMMICH_REPOSITORY_URL; + } + + if (process.env.IMMICH_BUILD) { + attributes['immich.build'] = process.env.IMMICH_BUILD; + } + + if (process.env.IMMICH_SOURCE_REF) { + attributes['immich.source_ref'] = process.env.IMMICH_SOURCE_REF; + } + + if (process.env.IMMICH_BUILD_IMAGE) { + attributes['immich.build_image'] = process.env.IMMICH_BUILD_IMAGE; + } + + const resource = resourceFromAttributes(attributes); + + const aggregationBoundaries = [ + 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000, + ]; + + // Use appropriate port based on worker type + const defaultPort = isForkedChild ? '8081' : '8082'; + const portEnvVar = isForkedChild ? 'IMMICH_API_METRICS_PORT' : 'IMMICH_MICROSERVICES_METRICS_PORT'; + const metricsPort = Number.parseInt(process.env[portEnvVar] || defaultPort, 10); + + const traceExporter = tracingEndpoint ? new OTLPTraceExporter({ url: `${tracingEndpoint}/v1/traces` }) : undefined; + const spanProcessor = traceExporter ? new FilteringSpanProcessor(new BatchSpanProcessor(traceExporter)) : undefined; + + const metricReaders = []; + if (metricsEnabled) { + metricReaders.push(new PrometheusExporter({ port: metricsPort })); + } + if (tracingEndpoint) { + metricReaders.push( + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ url: `${tracingEndpoint}/v1/metrics` }), + }), + ); + } + + const sdk = new NodeSDK({ + resource, + spanProcessors: spanProcessor ? [spanProcessor] : undefined, + metricReaders: metricReaders.length > 0 ? metricReaders : undefined, + views: metricsEnabled + ? [ + { + instrumentName: '*', + instrumentUnit: 'ms', + aggregation: { + type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, + options: { boundaries: aggregationBoundaries }, + }, + }, + ] + : undefined, + contextManager: new AsyncLocalStorageContextManager(), + instrumentations: [new HttpInstrumentation(), new IORedisInstrumentation(), new NestInstrumentation()], + }); + + sdk.start(); +} diff --git a/server/src/types.ts b/server/src/types.ts index f851ebd994..be20c09a52 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -10,6 +10,7 @@ import { AssetType, DatabaseSslMode, ExifOrientation, + GeneratedImageType, ImageFormat, JobName, MemoryType, @@ -64,7 +65,10 @@ export interface DecodeToBufferOptions extends DecodeImageOptions { orientation?: ExifOrientation; } -export type GenerateThumbnailOptions = Pick & DecodeToBufferOptions; +export type GenerateThumbnailOptions = Pick & + DecodeToBufferOptions & { + imageType?: GeneratedImageType; + }; export type GenerateThumbnailFromBufferOptions = GenerateThumbnailOptions & { raw: RawImageInfo }; @@ -178,8 +182,14 @@ export type ConcurrentQueueName = Exclude< export type Jobs = { [K in JobItem['name']]: (JobItem & { name: K })['data'] }; export type JobOf = Jobs[T]; +export interface TraceContext { + traceparent: string; + tracestate?: string; +} + export interface IBaseJob { force?: boolean; + traceContext?: TraceContext; } export interface IDelayedJob extends IBaseJob { diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index a041946a28..8a6ef544a5 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -1,6 +1,7 @@ import { AliasedRawBuilder, DeduplicateJoinsPlugin, + Dialect, Expression, ExpressionBuilder, ExpressionWrapper, @@ -22,6 +23,7 @@ import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } fr import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; import { DB } from 'src/schema'; import { DatabaseConnectionParams, VectorExtension } from 'src/types'; +import { createInstrumentedDriver } from 'src/utils/otel-kysely-plugin'; type Ssl = 'require' | 'allow' | 'prefer' | 'verify-full' | boolean | object; @@ -59,50 +61,65 @@ export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) => }; }; +const DATABASE_MAX_CONNECTIONS = 10; + +function createInstrumentedDialect(innerDialect: Dialect, maxConnections: number): Dialect { + return { + createAdapter: () => innerDialect.createAdapter(), + createDriver: () => createInstrumentedDriver(innerDialect.createDriver(), maxConnections), + createIntrospector: (db) => innerDialect.createIntrospector(db), + createQueryCompiler: () => innerDialect.createQueryCompiler(), + }; +} + export const getKyselyConfig = ( params: DatabaseConnectionParams, options: Partial>> = {}, ): KyselyConfig => { const config = asPostgresConnectionConfig(params); + const maxConnections = options.max ?? DATABASE_MAX_CONNECTIONS; + + const innerDialect = new PostgresJSDialect({ + postgres: postgres({ + onnotice: (notice: Notice) => { + if (notice['severity'] !== 'NOTICE') { + console.warn('Postgres notice:', notice); + } + }, + max: maxConnections, + types: { + date: { + to: 1184, + from: [1082, 1114, 1184], + serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x), + parse: (x: string) => new Date(x), + }, + bigint: { + to: 20, + from: [20, 1700], + parse: (value: string) => Number.parseInt(value), + serialize: (value: number) => value.toString(), + }, + }, + connection: { + TimeZone: 'UTC', + }, + host: config.host, + port: config.port, + username: config.username, + password: config.password, + database: config.database, + ssl: config.ssl, + ...options, + }), + }); return { - dialect: new PostgresJSDialect({ - postgres: postgres({ - onnotice: (notice: Notice) => { - if (notice['severity'] !== 'NOTICE') { - console.warn('Postgres notice:', notice); - } - }, - max: 10, - types: { - date: { - to: 1184, - from: [1082, 1114, 1184], - serialize: (x: Date | string) => (x instanceof Date ? x.toISOString() : x), - parse: (x: string) => new Date(x), - }, - bigint: { - to: 20, - from: [20, 1700], - parse: (value: string) => Number.parseInt(value), - serialize: (value: number) => value.toString(), - }, - }, - connection: { - TimeZone: 'UTC', - }, - host: config.host, - port: config.port, - username: config.username, - password: config.password, - database: config.database, - ssl: config.ssl, - ...options, - }), - }), - log(event) { + dialect: createInstrumentedDialect(innerDialect, maxConnections), + plugins: [], + log: (event) => { if (event.level === 'error') { - console.error('Query failed :', { + console.error('Query failed:', { durationMs: event.queryDurationMillis, error: event.error, sql: event.query.sql, diff --git a/server/src/utils/file.ts b/server/src/utils/file.ts index 6ec78b0f21..3267ac4341 100644 --- a/server/src/utils/file.ts +++ b/server/src/utils/file.ts @@ -1,13 +1,17 @@ import { HttpException, StreamableFile } from '@nestjs/common'; +import { context, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; import { NextFunction, Response } from 'express'; -import { access, constants } from 'node:fs/promises'; +import { access, constants, stat } from 'node:fs/promises'; import { basename, extname } from 'node:path'; +import { PassThrough } from 'node:stream'; import { promisify } from 'node:util'; import { CacheControl } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { ImmichReadStream } from 'src/repositories/storage.repository'; import { isConnectionAborted } from 'src/utils/misc'; +const fileTracer = trace.getTracer('immich-file'); + export function getFileNameWithoutExtension(path: string): string { return basename(path, extname(path)); } @@ -49,6 +53,8 @@ export const sendFile = async ( const _sendFile = (path: string, options: SendFileOptions) => promisify(res.sendFile).bind(res)(path, options); + const span = fileTracer.startSpan('file.send', { kind: SpanKind.INTERNAL }, context.active()); + try { const file = await handler(); const cacheControlHeader = cacheControlHeaders[file.cacheControl]; @@ -64,10 +70,24 @@ export const sendFile = async ( await access(file.path, constants.R_OK); - return await _sendFile(file.path, { dotfiles: 'allow' }); + // Get file size for telemetry + const fileStats = await stat(file.path); + + span.setAttribute('file.path', file.path); + span.setAttribute('file.size', fileStats.size); + span.setAttribute('file.content_type', file.contentType); + span.setAttribute('file.name', file.fileName || basename(file.path)); + span.setStatus({ code: SpanStatusCode.OK }); + + const result = await _sendFile(file.path, { dotfiles: 'allow' }); + span.end(); + return result; } catch (error: Error | any) { // ignore client-closed connection if (isConnectionAborted(error) || res.headersSent) { + span.setAttribute('file.connection_aborted', true); + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); return; } @@ -76,11 +96,46 @@ export const sendFile = async ( logger.error(`Unable to send file: ${error}`, error.stack); } + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + if (error instanceof Error) { + span.recordException(error); + } + span.end(); + res.header('Cache-Control', 'none'); next(error); } }; export const asStreamableFile = ({ stream, type, length }: ImmichReadStream) => { - return new StreamableFile(stream, { type, length }); + const span = fileTracer.startSpan('file.stream', { kind: SpanKind.INTERNAL }, context.active()); + + if (length !== undefined) { + span.setAttribute('file.size', length); + } + if (type) { + span.setAttribute('file.content_type', type); + } + + let bytesStreamed = 0; + + const trackedStream = new PassThrough(); + trackedStream.on('data', (chunk: Buffer) => { + bytesStreamed += chunk.length; + }); + trackedStream.on('end', () => { + span.setAttribute('file.bytes_streamed', bytesStreamed); + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + }); + trackedStream.on('error', (error: Error) => { + span.setAttribute('file.bytes_streamed', bytesStreamed); + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + span.recordException(error); + span.end(); + }); + + stream.pipe(trackedStream); + + return new StreamableFile(trackedStream, { type, length }); }; diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts new file mode 100644 index 0000000000..637a4a6938 --- /dev/null +++ b/server/src/utils/instrumentation.ts @@ -0,0 +1,35 @@ +import { context, Span, SpanKind, SpanStatusCode, trace } from '@opentelemetry/api'; + +const tracer = trace.getTracer('immich'); + +type TracedOptions = { + name: string; + kind?: SpanKind; +}; + +export function Traced(options: string | TracedOptions) { + const { name: spanName, kind = SpanKind.INTERNAL } = + typeof options === 'string' ? { name: options } : options; + + return function (_target: unknown, _propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: unknown[]) { + return tracer.startActiveSpan(spanName, { kind }, context.active(), async (span: Span) => { + try { + const result = await originalMethod.apply(this, args); + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span.recordException(error as Error); + throw error; + } finally { + span.end(); + } + }); + }; + + return descriptor; + }; +} diff --git a/server/src/utils/otel-kysely-plugin.ts b/server/src/utils/otel-kysely-plugin.ts new file mode 100644 index 0000000000..c8255eb789 --- /dev/null +++ b/server/src/utils/otel-kysely-plugin.ts @@ -0,0 +1,360 @@ +import { context, metrics, Span, SpanKind, SpanStatusCode, trace, ValueType } from '@opentelemetry/api'; +import type { CompiledQuery, DatabaseConnection, Driver, QueryResult, TransactionSettings } from 'kysely'; + +const dbTracer = trace.getTracer('immich-db'); +const instrumentedConnections = new WeakSet(); +const traceDbConnections = process.env.IMMICH_OTEL_TRACE_DB_CONNECTIONS === 'true'; + +let poolMetricsInitialized = false; + +function initializePoolMetrics(maxConnections: number) { + if (poolMetricsInitialized) { + return; + } + poolMetricsInitialized = true; + + const meter = metrics.getMeter('immich-db'); + + meter + .createObservableGauge('db.client.connections.max', { + description: 'Maximum number of connections in the pool', + unit: '{connection}', + valueType: ValueType.INT, + }) + .addCallback((observable) => { + observable.observe(maxConnections, { 'db.system': 'postgresql' }); + }); +} + +function getOperationName(sql: string): string { + const trimmed = sql.trimStart().toUpperCase(); + if (trimmed.startsWith('SELECT')) { + return 'SELECT'; + } + if (trimmed.startsWith('INSERT')) { + return 'INSERT'; + } + if (trimmed.startsWith('UPDATE')) { + return 'UPDATE'; + } + if (trimmed.startsWith('DELETE')) { + return 'DELETE'; + } + if (trimmed.startsWith('WITH')) { + return 'WITH'; + } + if (trimmed.startsWith('BEGIN')) { + return 'BEGIN'; + } + if (trimmed.startsWith('COMMIT')) { + return 'COMMIT'; + } + if (trimmed.startsWith('ROLLBACK')) { + return 'ROLLBACK'; + } + return 'QUERY'; +} + +function getTableName(sql: string): string | undefined { + const fromMatch = sql.match(/\bFROM\s+"?(\w+)"?/i); + if (fromMatch) { + return fromMatch[1]; + } + + const intoMatch = sql.match(/\bINTO\s+"?(\w+)"?/i); + if (intoMatch) { + return intoMatch[1]; + } + + const updateMatch = sql.match(/\bUPDATE\s+"?(\w+)"?/i); + if (updateMatch) { + return updateMatch[1]; + } + + return undefined; +} + +function createInstrumentedConnection( + connection: DatabaseConnection, + transactionSpan?: Span, +): DatabaseConnection { + if (instrumentedConnections.has(connection)) { + return connection; + } + + const instrumentedConnection: DatabaseConnection = { + async executeQuery(compiledQuery: CompiledQuery): Promise> { + const sql = compiledQuery.sql; + const operationName = getOperationName(sql); + const tableName = getTableName(sql); + const spanName = tableName ? `${operationName} ${tableName}` : operationName; + + const parentContext = transactionSpan + ? trace.setSpan(context.active(), transactionSpan) + : context.active(); + + return dbTracer.startActiveSpan(spanName, { kind: SpanKind.CLIENT }, parentContext, async (span) => { + try { + span.setAttribute('db.system', 'postgresql'); + span.setAttribute('db.operation.name', operationName); + span.setAttribute('db.query.text', sql); + + if (tableName) { + span.setAttribute('db.collection.name', tableName); + } + + if (compiledQuery.parameters.length > 0) { + span.setAttribute('db.query.parameter_count', compiledQuery.parameters.length); + } + + const result = await connection.executeQuery(compiledQuery); + + span.setAttribute('db.response.row_count', result.rows.length); + + if (result.numAffectedRows !== undefined) { + span.setAttribute('db.operation.affected_rows', Number(result.numAffectedRows)); + } + + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (error) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + span.recordException(error); + } + throw error; + } finally { + span.end(); + } + }); + }, + + async *streamQuery( + compiledQuery: CompiledQuery, + chunkSize: number, + ): AsyncIterableIterator> { + const sql = compiledQuery.sql; + const operationName = getOperationName(sql); + const tableName = getTableName(sql); + const spanName = tableName ? `STREAM ${operationName} ${tableName}` : `STREAM ${operationName}`; + + const parentContext = transactionSpan + ? trace.setSpan(context.active(), transactionSpan) + : context.active(); + + const span = dbTracer.startSpan(spanName, { kind: SpanKind.CLIENT }, parentContext); + + span.setAttribute('db.system', 'postgresql'); + span.setAttribute('db.operation.name', operationName); + span.setAttribute('db.query.text', sql); + span.setAttribute('db.stream.chunk_size', chunkSize); + + if (tableName) { + span.setAttribute('db.collection.name', tableName); + } + + let totalRows = 0; + let chunkCount = 0; + + try { + for await (const chunk of connection.streamQuery(compiledQuery, chunkSize)) { + chunkCount++; + totalRows += chunk.rows.length; + yield chunk; + } + + span.setAttribute('db.stream.chunk_count', chunkCount); + span.setAttribute('db.stream.total_rows', totalRows); + span.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + span.recordException(error); + } + throw error; + } finally { + span.end(); + } + }, + }; + + instrumentedConnections.add(instrumentedConnection); + return instrumentedConnection; +} + +const transactionSpans = new WeakMap(); +const unwrappedConnections = new WeakMap(); + +/** + * Creates an instrumented driver that wraps a Kysely driver with OpenTelemetry tracing. + * Provides: + * - Query spans with row counts and affected rows + * - Streaming query spans with chunk metrics + * - Transaction spans that parent query spans + * - Connection acquisition timing + */ +export function createInstrumentedDriver(driver: Driver, maxConnections: number): Driver { + initializePoolMetrics(maxConnections); + + return new Proxy(driver, { + get(target, prop, receiver) { + if (prop === 'init' || prop === 'destroy') { + const method = Reflect.get(target, prop, target); + if (typeof method === 'function') { + return method.bind(target); + } + return method; + } + + if (prop === 'acquireConnection') { + return async function (): Promise { + if (!traceDbConnections) { + const connection = await target.acquireConnection(); + const instrumentedConnection = createInstrumentedConnection(connection); + unwrappedConnections.set(instrumentedConnection, connection); + return instrumentedConnection; + } + + return dbTracer.startActiveSpan( + 'DB Connection Acquire', + { kind: SpanKind.CLIENT }, + context.active(), + async (span) => { + try { + span.setAttribute('db.system', 'postgresql'); + + const connection = await target.acquireConnection(); + const instrumentedConnection = createInstrumentedConnection(connection); + + unwrappedConnections.set(instrumentedConnection, connection); + + span.setStatus({ code: SpanStatusCode.OK }); + return instrumentedConnection; + } catch (error) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + span.recordException(error); + } + throw error; + } finally { + span.end(); + } + }, + ); + }; + } + + if (prop === 'beginTransaction') { + return async function ( + connection: DatabaseConnection, + settings: TransactionSettings, + ): Promise { + const unwrapped = unwrappedConnections.get(connection) ?? connection; + + const transactionSpan = dbTracer.startSpan( + 'DB Transaction', + { kind: SpanKind.CLIENT }, + context.active(), + ); + transactionSpan.setAttribute('db.system', 'postgresql'); + transactionSpan.setAttribute('db.operation.name', 'TRANSACTION'); + + if (settings.isolationLevel) { + transactionSpan.setAttribute('db.transaction.isolation_level', settings.isolationLevel); + } + + transactionSpans.set(connection, transactionSpan); + + const instrumentedForTransaction = createInstrumentedConnection(unwrapped, transactionSpan); + unwrappedConnections.set(instrumentedForTransaction, unwrapped); + + Object.assign(connection, instrumentedForTransaction); + + try { + await target.beginTransaction(unwrapped, settings); + } catch (error) { + transactionSpan.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + transactionSpan.recordException(error); + } + transactionSpan.end(); + transactionSpans.delete(connection); + throw error; + } + }; + } + + if (prop === 'commitTransaction') { + return async function (connection: DatabaseConnection): Promise { + const unwrapped = unwrappedConnections.get(connection) ?? connection; + const transactionSpan = transactionSpans.get(connection); + + try { + await target.commitTransaction(unwrapped); + transactionSpan?.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + transactionSpan?.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + transactionSpan?.recordException(error); + } + throw error; + } finally { + transactionSpan?.end(); + transactionSpans.delete(connection); + } + }; + } + + if (prop === 'rollbackTransaction') { + return async function (connection: DatabaseConnection): Promise { + const unwrapped = unwrappedConnections.get(connection) ?? connection; + const transactionSpan = transactionSpans.get(connection); + + try { + await target.rollbackTransaction(unwrapped); + transactionSpan?.setAttribute('db.transaction.rollback', true); + transactionSpan?.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + transactionSpan?.setStatus({ + code: SpanStatusCode.ERROR, + message: error instanceof Error ? error.message : String(error), + }); + if (error instanceof Error) { + transactionSpan?.recordException(error); + } + throw error; + } finally { + transactionSpan?.end(); + transactionSpans.delete(connection); + } + }; + } + + if (prop === 'releaseConnection') { + return async function (connection: DatabaseConnection): Promise { + const unwrapped = unwrappedConnections.get(connection) ?? connection; + unwrappedConnections.delete(connection); + transactionSpans.delete(connection); + return target.releaseConnection(unwrapped); + }; + } + + return Reflect.get(target, prop, receiver); + }, + }); +} diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index 8f06b4b0b1..5f49f7cfae 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -1,3 +1,6 @@ +// Must be first import - sets up OpenTelemetry SDK before any modules load +import 'src/telemetry-preload'; + import { NestFactory } from '@nestjs/core'; import { isMainThread } from 'node:worker_threads'; import { MicroservicesModule } from 'src/app.module';