cache converted images

backend/src/collections/image-db/image-file-db.service.ts

@@ -77,7 +77,7 @@ export class ImageFileDBService {
     key: string,
     mime: string,
     file: Buffer,
-  ): AsyncFailable<true> {
+  ): AsyncFailable<EImageDerivativeBackend> {
     const imageDerivative = new EImageDerivativeBackend();
     imageDerivative.imageId = imageId;
     imageDerivative.key = key;
@@ -85,25 +85,20 @@ export class ImageFileDBService {
     imageDerivative.data = file;
 
     try {
-      await this.imageDerivativeRepo.save(imageDerivative);
+      return await this.imageDerivativeRepo.save(imageDerivative);
     } catch (e) {
       return Fail(e);
     }
-
-    return true;
   }
 
   public async getDerivative(
     imageId: string,
     key: string,
-  ): AsyncFailable<EImageDerivativeBackend> {
+  ): AsyncFailable<EImageDerivativeBackend | null> {
     try {
-      const found = await this.imageDerivativeRepo.findOne({
+      return await this.imageDerivativeRepo.findOne({
         where: { imageId, key },
       });
-
-      if (!found) return Fail('Image not found');
-      return found;
     } catch (e) {
       return Fail(e);
     }

backend/src/collections/preference-db/sys-preference-db.service.ts

@@ -63,32 +63,34 @@ export class SysPreferenceService {
     let validatedKey = this.prefCommon.validatePrefKey(key, SysPreference);
     if (HasFailed(validatedKey)) return validatedKey;
 
-    let foundSysPreference: ESysPreferenceBackend;
+    return MutexFallBack(
-    try {
+      'fetchSysPrefrence',
-      foundSysPreference = await MutexFallBack(
+      async () => {
-        'fetchSysPrefrence',
+        let existing: ESysPreferenceBackend | null;
-        () =>
+        try {
-          this.sysPreferenceRepository.findOne({
+          existing = await this.sysPreferenceRepository.findOne({
             where: { key: validatedKey as SysPreference },
             cache: 60000,
-          }),
+          });
-        () => this.saveDefault(validatedKey as SysPreference),
+          if (!existing) return null;
-      );
+        } catch (e) {
-    } catch (e) {
+          return Fail(e);
-      return Fail(e);
+        }
-    }
 
-    // Validate
+        // Validate
-    const result = ESysPreferenceSchema.safeParse(foundSysPreference);
+        const result = ESysPreferenceSchema.safeParse(existing);
-    if (!result.success) {
+        if (!result.success) {
-      return Fail(result.error);
+          return Fail(result.error);
-    }
+        }
 
-    // Return
+        // Return
-    return this.prefCommon.validateAndUnpackPref(
+        return this.prefCommon.validateAndUnpackPref(
-      result.data,
+          result.data,
-      SysPreference,
+          SysPreference,
-      SysPreferenceValueTypes,
+          SysPreferenceValueTypes,
+        );
+      },
+      () => this.saveDefault(validatedKey as SysPreference),
     );
   }
 

backend/src/collections/preference-db/usr-preference-db.service.ts

@@ -68,38 +68,40 @@ export class UsrPreferenceService {
     let validatedKey = this.prefCommon.validatePrefKey(key, UsrPreference);
     if (HasFailed(validatedKey)) return validatedKey;
 
-    let foundUsrPreference: EUsrPreferenceBackend;
+    return MutexFallBack(
-    try {
+      'fetchUsrPrefrence',
-      foundUsrPreference = await MutexFallBack(
+      async () => {
-        'fetchUsrPrefrence',
+        let existing: EUsrPreferenceBackend | null;
-        () =>
+        try {
-          this.usrPreferenceRepository.findOne({
+          existing = await this.usrPreferenceRepository.findOne({
             where: { key: validatedKey as UsrPreference, userId: userid },
             cache: 60000,
-          }),
+          });
-        () => this.saveDefault(userid, validatedKey as UsrPreference),
+          if (!existing) return null;
-      );
+        } catch (e) {
-    } catch (e) {
+          return Fail(e);
-      return Fail(e);
+        }
-    }
 
-    // Validate
+        // Validate
-    const result = EUsrPreferenceSchema.safeParse(foundUsrPreference);
+        const result = EUsrPreferenceSchema.safeParse(existing);
-    if (!result.success) {
+        if (!result.success) {
-      return Fail(result.error);
+          return Fail(result.error);
-    }
+        }
 
-    // Return
+        // Return
-    const unpacked = this.prefCommon.validateAndUnpackPref(
+        const unpacked = this.prefCommon.validateAndUnpackPref(
-      result.data,
+          result.data,
-      UsrPreference,
+          UsrPreference,
-      UsrPreferenceValueTypes,
+          UsrPreferenceValueTypes,
+        );
+        if (HasFailed(unpacked)) return unpacked;
+        return {
+          ...unpacked,
+          user: result.data.userId,
+        };
+      },
+      () => this.saveDefault(userid, validatedKey as UsrPreference),
     );
-    if (HasFailed(unpacked)) return unpacked;
-    return {
-      ...unpacked,
-      user: result.data.userId,
-    };
   }
 
   public async getStringPreference(

backend/src/managers/image/image.service.ts

@@ -1,4 +1,5 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
+import Crypto from 'crypto';
 import { fileTypeFromBuffer, FileTypeResult } from 'file-type';
 import { FullMime } from 'picsur-shared/dist/dto/mimes.dto';
 import { UsrPreference } from 'picsur-shared/dist/dto/usr-preferences.dto';
@@ -12,15 +13,14 @@ import { ImageFileType } from '../../models/constants/image-file-types.const';
 import { EImageDerivativeBackend } from '../../models/entities/image-derivative.entity';
 import { EImageFileBackend } from '../../models/entities/image-file.entity';
 import { EImageBackend } from '../../models/entities/image.entity';
+import { MutexFallBack } from '../../models/util/mutex-fallback';
 import { ImageConverterService } from './image-converter.service';
 import { ImageProcessorService } from './image-processor.service';
 
-// Right now this service is mostly a wrapper for the imagedbservice.
-// But in the future the actual image logic will happend here
-// And the image storing part will stay in the imagedbservice
-
 @Injectable()
 export class ImageManagerService {
+  private readonly logger = new Logger(ImageManagerService.name);
+
   constructor(
     private readonly imagesService: ImageDBService,
     private readonly imageFilesService: ImageFileDBService,
@@ -84,31 +84,47 @@ export class ImageManagerService {
 
   public async getConverted(
     imageId: string,
-    mime: string,
+    options: {
+      mime: string;
+    },
   ): AsyncFailable<EImageDerivativeBackend> {
-    const targetMime = ParseMime(mime);
+    const targetMime = ParseMime(options.mime);
     if (HasFailed(targetMime)) return targetMime;
-    
-    const masterImage = await this.getMaster(imageId);
-    if (HasFailed(masterImage)) return masterImage;
 
-    const sourceMime = ParseMime(masterImage.mime);
+    const converted_key = this.getConvertHash(options);
-    if (HasFailed(sourceMime)) return sourceMime;
 
-    const convertResult = await this.convertService.convert(
+    return MutexFallBack(
-      masterImage.data,
+      converted_key,
-      sourceMime,
+      () => this.imageFilesService.getDerivative(imageId, converted_key),
-      targetMime,
+      async () => {
+        const masterImage = await this.getMaster(imageId);
+        if (HasFailed(masterImage)) return masterImage;
+
+        const sourceMime = ParseMime(masterImage.mime);
+        if (HasFailed(sourceMime)) return sourceMime;
+
+        const startTime = Date.now();
+        const convertResult = await this.convertService.convert(
+          masterImage.data,
+          sourceMime,
+          targetMime,
+        );
+        if (HasFailed(convertResult)) return convertResult;
+
+        this.logger.verbose(
+          `Converted ${imageId} from ${sourceMime.mime} to ${
+            targetMime.mime
+          } in ${Date.now() - startTime}ms`,
+        );
+
+        return await this.imageFilesService.addDerivative(
+          imageId,
+          converted_key,
+          convertResult.mime,
+          convertResult.image,
+        );
+      },
     );
-    if (HasFailed(convertResult)) return convertResult;
-
-    const returned = new EImageDerivativeBackend();
-    returned.data = convertResult.image;
-    returned.mime = convertResult.mime;
-    returned.imageId = imageId;
-    returned.key = 'aight';
-
-    return returned;
   }
 
   // File getters ==============================================================
@@ -158,4 +174,13 @@ export class ImageManagerService {
     const fullMime = ParseMime(mime ?? 'other/unknown');
     return fullMime;
   }
+
+  private getConvertHash(options: object) {
+    // Return a sha256 hash of the stringified options
+    const stringified = JSON.stringify(options);
+    const hash = Crypto.createHash('sha256');
+    hash.update(stringified);
+    const digest = hash.digest('hex');
+    return digest;
+  }
 }

backend/src/models/util/mutex-fallback.ts

@@ -14,7 +14,7 @@ const fallBackMap: Record<string, Promise<unknown>> = {};
 
 export async function MutexFallBack<
   MF extends () => Promise<O | null | undefined>,
-  FF extends () => Promise<unknown>,
+  FF extends () => Promise<O>,
   O,
 >(key: string, mainFunc: MF, fallBackFunc: FF): Promise<O> {
   const try_it = await mainFunc();
@@ -25,7 +25,9 @@ export async function MutexFallBack<
     return MutexFallBack(key, mainFunc, fallBackFunc);
   }
 
-  fallBackMap[key] = fallBackFunc();
+  const fallBackPromise = fallBackFunc();
+
+  fallBackMap[key] = fallBackPromise;
   fallBackMap[key]
     .then(() => {
       delete fallBackMap[key];
@@ -33,5 +35,6 @@ export async function MutexFallBack<
     .catch(() => {
       delete fallBackMap[key];
     });
-  return MutexFallBack(key, mainFunc, fallBackFunc);
+
+  return fallBackPromise;
 }

backend/src/routes/image/image.controller.ts

@@ -5,7 +5,8 @@ import {
   InternalServerErrorException,
   Logger,
   NotFoundException,
-  Post, Res
+  Post,
+  Res
 } from '@nestjs/common';
 import { FastifyReply } from 'fastify';
 import { ImageMetaResponse } from 'picsur-shared/dist/dto/api/image.dto';
@@ -47,7 +48,9 @@ export class ImageController {
       return image.data;
     }
 
-    const image = await this.imagesService.getConverted(fullid.id, fullid.mime);
+    const image = await this.imagesService.getConverted(fullid.id, {
+      mime: fullid.mime,
+    });
     if (HasFailed(image)) {
       this.logger.warn(image.getReason());
       throw new NotFoundException('Could not find image');