fix: asset update race condition (#24384)

* fix: asset update race condition

* fix: asset update race condition

* single statement

* update sql

* missed one

* fix `none` handling

* fix: tests

* chore: simplify update all assets

* fix: updating lockable properties

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

server/src/services/asset-media.service.ts

@@ -370,7 +370,10 @@ export class AssetMediaService extends BaseService {
       : this.assetRepository.deleteFile({ assetId, type: AssetFileType.Sidecar }));
 
     await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
-    await this.assetRepository.upsertExif({ assetId, fileSizeInByte: file.size });
+    await this.assetRepository.upsertExif(
+      { assetId, fileSizeInByte: file.size },
+      { lockedPropertiesBehavior: 'override' },
+    );
     await this.jobRepository.queue({
       name: JobName.AssetExtractMetadata,
       data: { id: assetId, source: 'upload' },
@@ -399,7 +402,10 @@ export class AssetMediaService extends BaseService {
     });
 
     const { size } = await this.storageRepository.stat(created.originalPath);
-    await this.assetRepository.upsertExif({ assetId: created.id, fileSizeInByte: size });
+    await this.assetRepository.upsertExif(
+      { assetId: created.id, fileSizeInByte: size },
+      { lockedPropertiesBehavior: 'override' },
+    );
     await this.jobRepository.queue({ name: JobName.AssetExtractMetadata, data: { id: created.id, source: 'copy' } });
     return created;
   }
@@ -440,7 +446,10 @@ export class AssetMediaService extends BaseService {
       await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));
     }
     await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
-    await this.assetRepository.upsertExif({ assetId: asset.id, fileSizeInByte: file.size });
+    await this.assetRepository.upsertExif(
+      { assetId: asset.id, fileSizeInByte: file.size },
+      { lockedPropertiesBehavior: 'override' },
+    );
 
     await this.eventRepository.emit('AssetCreate', { asset });
 

server/src/services/asset.service.spec.ts

@@ -225,7 +225,10 @@ describe(AssetService.name, () => {
 
       await sut.update(authStub.admin, 'asset-1', { description: 'Test description' });
 
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' });
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+        { assetId: 'asset-1', description: 'Test description', lockedProperties: ['description'] },
+        { lockedPropertiesBehavior: 'append' },
+      );
     });
 
     it('should update the exif rating', async () => {
@@ -235,7 +238,14 @@ describe(AssetService.name, () => {
 
       await sut.update(authStub.admin, 'asset-1', { rating: 3 });
 
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 });
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+        {
+          assetId: 'asset-1',
+          rating: 3,
+          lockedProperties: ['rating'],
+        },
+        { lockedPropertiesBehavior: 'append' },
+      );
     });
 
     it('should fail linking a live video if the motion part could not be found', async () => {
@@ -427,9 +437,7 @@ describe(AssetService.name, () => {
       });
       expect(mocks.asset.updateAll).toHaveBeenCalled();
       expect(mocks.asset.updateAllExif).toHaveBeenCalledWith(['asset-1'], { latitude: 0, longitude: 0 });
-      expect(mocks.job.queueAll).toHaveBeenCalledWith([
-        { name: JobName.SidecarWrite, data: { id: 'asset-1', latitude: 0, longitude: 0 } },
-      ]);
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
     });
 
     it('should update exif table if latitude field is provided', async () => {
@@ -450,9 +458,7 @@ describe(AssetService.name, () => {
         latitude: 30,
         longitude: 50,
       });
-      expect(mocks.job.queueAll).toHaveBeenCalledWith([
-        { name: JobName.SidecarWrite, data: { id: 'asset-1', dateTimeOriginal, latitude: 30, longitude: 50 } },
-      ]);
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
     });
 
     it('should update Assets table if duplicateId is provided as null', async () => {
@@ -482,18 +488,7 @@ describe(AssetService.name, () => {
         timeZone,
       });
       expect(mocks.asset.updateDateTimeOriginal).toHaveBeenCalledWith(['asset-1'], dateTimeRelative, timeZone);
-      expect(mocks.job.queueAll).toHaveBeenCalledWith([
-        {
-          name: JobName.SidecarWrite,
-          data: {
-            id: 'asset-1',
-            dateTimeOriginal: '2020-02-25T06:41:00.000+02:00',
-            description: undefined,
-            latitude: undefined,
-            longitude: undefined,
-          },
-        },
-      ]);
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
     });
   });
 

server/src/services/asset.service.ts

@@ -30,9 +30,10 @@ import {
   QueueName,
 } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
-import { ISidecarWriteJob, JobItem, JobOf } from 'src/types';
+import { JobItem, JobOf } from 'src/types';
 import { requireElevatedPermission } from 'src/utils/access';
 import { getAssetFiles, getMyPartnerIds, onAfterUnlink, onBeforeLink, onBeforeUnlink } from 'src/utils/asset.util';
+import { updateLockedColumns } from 'src/utils/database';
 
 @Injectable()
 export class AssetService extends BaseService {
@@ -142,56 +143,26 @@ export class AssetService extends BaseService {
     } = dto;
     await this.requireAccess({ auth, permission: Permission.AssetUpdate, ids });
 
-    const assetDto = { isFavorite, visibility, duplicateId };
-    const exifDto = { latitude, longitude, rating, description, dateTimeOriginal };
+    const assetDto = _.omitBy({ isFavorite, visibility, duplicateId }, _.isUndefined);
+    const exifDto = _.omitBy({ latitude, longitude, rating, description, dateTimeOriginal }, _.isUndefined);
 
-    const isExifChanged = Object.values(exifDto).some((v) => v !== undefined);
-    if (isExifChanged) {
+    if (Object.keys(exifDto).length > 0) {
       await this.assetRepository.updateAllExif(ids, exifDto);
     }
 
-    const assets =
-      (dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined
-        ? await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone)
-        : undefined;
-
-    const dateTimesWithTimezone = assets
-      ? assets.map((asset) => {
-          const isoString = asset.dateTimeOriginal?.toISOString();
-          let dateTime = isoString ? DateTime.fromISO(isoString) : null;
-
-          if (dateTime && asset.timeZone) {
-            dateTime = dateTime.setZone(asset.timeZone);
-          }
-
-          return {
-            assetId: asset.assetId,
-            dateTimeOriginal: dateTime?.toISO() ?? null,
-          };
-        })
-      : ids.map((id) => ({ assetId: id, dateTimeOriginal }));
-
-    if (dateTimesWithTimezone.length > 0) {
-      await this.jobRepository.queueAll(
-        dateTimesWithTimezone.map(({ assetId: id, dateTimeOriginal }) => ({
-          name: JobName.SidecarWrite,
-          data: {
-            ...exifDto,
-            id,
-            dateTimeOriginal: dateTimeOriginal ?? undefined,
-          },
-        })),
-      );
+    if ((dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined) {
+      await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone);
     }
 
-    const isAssetChanged = Object.values(assetDto).some((v) => v !== undefined);
-    if (isAssetChanged) {
+    if (Object.keys(assetDto).length > 0) {
       await this.assetRepository.updateAll(ids, assetDto);
-
-      if (visibility === AssetVisibility.Locked) {
-        await this.albumRepository.removeAssetsFromAll(ids);
-      }
     }
+
+    if (visibility === AssetVisibility.Locked) {
+      await this.albumRepository.removeAssetsFromAll(ids);
+    }
+
+    await this.jobRepository.queueAll(ids.map((id) => ({ name: JobName.SidecarWrite, data: { id } })));
   }
 
   async copy(
@@ -456,12 +427,25 @@ export class AssetService extends BaseService {
     return asset;
   }
 
-  private async updateExif(dto: ISidecarWriteJob) {
+  private async updateExif(dto: {
+    id: string;
+    description?: string;
+    dateTimeOriginal?: string;
+    latitude?: number;
+    longitude?: number;
+    rating?: number;
+  }) {
     const { id, description, dateTimeOriginal, latitude, longitude, rating } = dto;
     const writes = _.omitBy({ description, dateTimeOriginal, latitude, longitude, rating }, _.isUndefined);
     if (Object.keys(writes).length > 0) {
-      await this.assetRepository.upsertExif({ assetId: id, ...writes });
-      await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id, ...writes } });
+      await this.assetRepository.upsertExif(
+        updateLockedColumns({
+          assetId: id,
+          ...writes,
+        }),
+        { lockedPropertiesBehavior: 'append' },
+      );
+      await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });
     }
   }
 }

server/src/services/metadata.service.spec.ts

@@ -187,7 +187,9 @@ describe(MetadataService.name, () => {
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.sidecar.id);
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }));
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }), {
+        lockedPropertiesBehavior: 'skip',
+      });
       expect(mocks.asset.update).toHaveBeenCalledWith(
         expect.objectContaining({
           id: assetStub.image.id,
@@ -214,6 +216,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
       expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
         expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
+        { lockedPropertiesBehavior: 'skip' },
       );
       expect(mocks.asset.update).toHaveBeenCalledWith({
         id: assetStub.image.id,
@@ -238,7 +241,10 @@ describe(MetadataService.name, () => {
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: fileCreatedAt }));
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+        expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
+        { lockedPropertiesBehavior: 'skip' },
+      );
       expect(mocks.asset.update).toHaveBeenCalledWith({
         id: assetStub.image.id,
         duration: null,
@@ -258,6 +264,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
 
       expect(mocks.asset.update).toHaveBeenCalledWith(
@@ -281,7 +288,9 @@ describe(MetadataService.name, () => {
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }));
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }), {
+        lockedPropertiesBehavior: 'skip',
+      });
       expect(mocks.asset.update).toHaveBeenCalledWith({
         id: assetStub.image.id,
         duration: null,
@@ -310,6 +319,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
       expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
         expect.objectContaining({ city: null, state: null, country: null }),
+        { lockedPropertiesBehavior: 'skip' },
       );
       expect(mocks.asset.update).toHaveBeenCalledWith({
         id: assetStub.withLocation.id,
@@ -339,6 +349,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
       expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
         expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
+        { lockedPropertiesBehavior: 'skip' },
       );
       expect(mocks.asset.update).toHaveBeenCalledWith({
         id: assetStub.withLocation.id,
@@ -358,7 +369,10 @@ describe(MetadataService.name, () => {
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ latitude: null, longitude: null }));
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+        expect.objectContaining({ latitude: null, longitude: null }),
+        { lockedPropertiesBehavior: 'skip' },
+      );
     });
 
     it('should extract tags from TagsList', async () => {
@@ -571,6 +585,7 @@ describe(MetadataService.name, () => {
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.video.id);
       expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
         expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -879,37 +894,40 @@ describe(MetadataService.name, () => {
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
       expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith({
-        assetId: assetStub.image.id,
-        bitsPerSample: expect.any(Number),
-        autoStackId: null,
-        colorspace: tags.ColorSpace,
-        dateTimeOriginal: dateForTest,
-        description: tags.ImageDescription,
-        exifImageHeight: null,
-        exifImageWidth: null,
-        exposureTime: tags.ExposureTime,
-        fNumber: null,
-        fileSizeInByte: 123_456,
-        focalLength: tags.FocalLength,
-        fps: null,
-        iso: tags.ISO,
-        latitude: null,
-        lensModel: tags.LensModel,
-        livePhotoCID: tags.MediaGroupUUID,
-        longitude: null,
-        make: tags.Make,
-        model: tags.Model,
-        modifyDate: expect.any(Date),
-        orientation: tags.Orientation?.toString(),
-        profileDescription: tags.ProfileDescription,
-        projectionType: 'EQUIRECTANGULAR',
-        timeZone: tags.tz,
-        rating: tags.Rating,
-        country: null,
-        state: null,
-        city: null,
-      });
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+        {
+          assetId: assetStub.image.id,
+          bitsPerSample: expect.any(Number),
+          autoStackId: null,
+          colorspace: tags.ColorSpace,
+          dateTimeOriginal: dateForTest,
+          description: tags.ImageDescription,
+          exifImageHeight: null,
+          exifImageWidth: null,
+          exposureTime: tags.ExposureTime,
+          fNumber: null,
+          fileSizeInByte: 123_456,
+          focalLength: tags.FocalLength,
+          fps: null,
+          iso: tags.ISO,
+          latitude: null,
+          lensModel: tags.LensModel,
+          livePhotoCID: tags.MediaGroupUUID,
+          longitude: null,
+          make: tags.Make,
+          model: tags.Model,
+          modifyDate: expect.any(Date),
+          orientation: tags.Orientation?.toString(),
+          profileDescription: tags.ProfileDescription,
+          projectionType: 'EQUIRECTANGULAR',
+          timeZone: tags.tz,
+          rating: tags.Rating,
+          country: null,
+          state: null,
+          city: null,
+        },
+        { lockedPropertiesBehavior: 'skip' },
+      );
       expect(mocks.asset.update).toHaveBeenCalledWith(
         expect.objectContaining({
           id: assetStub.image.id,
@@ -943,6 +961,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           timeZone: 'UTC+0',
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1103,6 +1122,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           description: '',
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
 
       mockReadTags({ ImageDescription: ' my\n description' });
@@ -1111,6 +1131,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           description: 'my\n description',
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1123,6 +1144,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           description: '1000',
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1346,6 +1368,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           modifyDate: expect.any(Date),
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1358,6 +1381,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           rating: null,
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1370,6 +1394,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           rating: 5,
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1382,6 +1407,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           rating: -1,
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
 
@@ -1503,7 +1529,9 @@ describe(MetadataService.name, () => {
       mockReadTags(exif);
 
       await sut.handleMetadataExtraction({ id: assetStub.image.id });
-      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected));
+      expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected), {
+        lockedPropertiesBehavior: 'skip',
+      });
     });
 
     it.each([
@@ -1529,6 +1557,7 @@ describe(MetadataService.name, () => {
         expect.objectContaining({
           lensModel: expected,
         }),
+        { lockedPropertiesBehavior: 'skip' },
       );
     });
   });
@@ -1637,12 +1666,14 @@ describe(MetadataService.name, () => {
 
   describe('handleSidecarWrite', () => {
     it('should skip assets that no longer exist', async () => {
+      mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
       mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);
       await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.Failed);
       expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
     });
 
     it('should skip jobs with no metadata', async () => {
+      mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
       const asset = factory.jobAssets.sidecarWrite();
       mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
       await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
@@ -1655,20 +1686,22 @@ describe(MetadataService.name, () => {
       const gps = 12;
       const date = '2023-11-22T04:56:12.196Z';
 
+      mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([
+        'description',
+        'latitude',
+        'longitude',
+        'dateTimeOriginal',
+      ]);
       mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
       await expect(
         sut.handleSidecarWrite({
           id: asset.id,
-          description,
-          latitude: gps,
-          longitude: gps,
-          dateTimeOriginal: date,
         }),
       ).resolves.toBe(JobStatus.Success);
       expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {
+        DateTimeOriginal: date,
         Description: description,
         ImageDescription: description,
-        DateTimeOriginal: date,
         GPSLatitude: gps,
         GPSLongitude: gps,
       });

server/src/services/metadata.service.ts

@@ -290,7 +290,7 @@ export class MetadataService extends BaseService {
     };
 
     const promises: Promise<unknown>[] = [
-      this.assetRepository.upsertExif(exifData),
+      this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' }),
       this.assetRepository.update({
         id: asset.id,
         duration: this.getDuration(exifTags),
@@ -393,22 +393,34 @@ export class MetadataService extends BaseService {
 
   @OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar })
   async handleSidecarWrite(job: JobOf<JobName.SidecarWrite>): Promise<JobStatus> {
-    const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;
+    const { id, tags } = job;
     const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
     if (!asset) {
       return JobStatus.Failed;
     }
 
+    const lockedProperties = await this.assetJobRepository.getLockedPropertiesForMetadataExtraction(id);
     const tagsList = (asset.tags || []).map((tag) => tag.value);
 
     const { sidecarFile } = getAssetFiles(asset.files);
     const sidecarPath = sidecarFile?.path || `${asset.originalPath}.xmp`;
 
+    const { description, dateTimeOriginal, latitude, longitude, rating } = _.pick(
+      {
+        description: asset.exifInfo.description,
+        dateTimeOriginal: asset.exifInfo.dateTimeOriginal,
+        latitude: asset.exifInfo.latitude,
+        longitude: asset.exifInfo.longitude,
+        rating: asset.exifInfo.rating,
+      },
+      lockedProperties,
+    );
+
     const exif = _.omitBy(
       <Tags>{
         Description: description,
         ImageDescription: description,
-        DateTimeOriginal: dateTimeOriginal,
+        DateTimeOriginal: dateTimeOriginal ? String(dateTimeOriginal) : undefined,
         GPSLatitude: latitude,
         GPSLongitude: longitude,
         Rating: rating,