add seperate collection for image files and meta

backend/src/collections/image-db/image-db.module.ts

@@ -1,11 +1,13 @@
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { EImageFileBackend } from '../../models/entities/image-file.entity';
 import { EImageBackend } from '../../models/entities/image.entity';
 import { ImageDBService } from './image-db.service';
+import { ImageFileDBService } from './image-file-db.service';
 
 @Module({
-  imports: [TypeOrmModule.forFeature([EImageBackend])],
+  imports: [TypeOrmModule.forFeature([EImageBackend, EImageFileBackend])],
-  providers: [ImageDBService],
+  providers: [ImageDBService, ImageFileDBService],
-  exports: [ImageDBService],
+  exports: [ImageDBService, ImageFileDBService],
 })
 export class ImageDBModule {}

backend/src/collections/image-db/image-db.service.ts

@@ -1,30 +1,25 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import {
+import { AsyncFailable, Fail } from 'picsur-shared/dist/types';
-  AsyncFailable,
-  Fail
-} from 'picsur-shared/dist/types';
 import { Repository } from 'typeorm';
+import { EImageFileBackend } from '../../models/entities/image-file.entity';
 import { EImageBackend } from '../../models/entities/image.entity';
-import { GetCols } from '../../models/util/collection';
 
 @Injectable()
 export class ImageDBService {
   constructor(
     @InjectRepository(EImageBackend)
-    private imageRepository: Repository<EImageBackend>,
+    private imageRepo: Repository<EImageBackend>,
+
+    @InjectRepository(EImageFileBackend)
+    private imageFileRepo: Repository<EImageFileBackend>,
   ) {}
 
-  public async create(
+  public async create(): AsyncFailable<EImageBackend> {
-    image: Buffer,
-    type: string,
-  ): AsyncFailable<EImageBackend> {
     let imageEntity = new EImageBackend();
-    imageEntity.data = image;
-    imageEntity.mime = type;
 
     try {
-      imageEntity = await this.imageRepository.save(imageEntity);
+      imageEntity = await this.imageRepo.save(imageEntity);
     } catch (e) {
       return Fail(e);
     }
@@ -32,22 +27,14 @@ export class ImageDBService {
     return imageEntity;
   }
 
-  public async findOne<B extends true | undefined = undefined>(
+  public async findOne(id: string): AsyncFailable<EImageBackend> {
-    id: string,
-    getPrivate?: B,
-  ): AsyncFailable<
-    B extends undefined ? EImageBackend : Required<EImageBackend>
-  > {
     try {
-      const found = await this.imageRepository.findOne({
+      const found = await this.imageRepo.findOne({
         where: { id },
-        select: getPrivate ? GetCols(this.imageRepository) : undefined,
       });
 
       if (!found) return Fail('Image not found');
-      return found as B extends undefined
+      return found;
-        ? EImageBackend
-        : Required<EImageBackend>;
     } catch (e) {
       return Fail(e);
     }
@@ -61,7 +48,7 @@ export class ImageDBService {
     if (count > 100) return Fail('Too many results');
 
     try {
-      const found = await this.imageRepository.find({
+      const found = await this.imageRepo.find({
         skip: count * page,
         take: count,
       });
@@ -75,8 +62,13 @@ export class ImageDBService {
 
   public async delete(id: string): AsyncFailable<true> {
     try {
-      const result = await this.imageRepository.delete({ id });
+      const filesResult = await this.imageFileRepo.delete({
-      if (result.affected === 0) return Fail('Image not found');
+        imageId: id,
+      });
+      const result = await this.imageRepo.delete({ id });
+      
+      if (result.affected === 0 && filesResult.affected === 0)
+        return Fail('Image not found');
     } catch (e) {
       return Fail(e);
     }
@@ -88,7 +80,8 @@ export class ImageDBService {
       return Fail('You must confirm that you want to delete all images');
 
     try {
-      await this.imageRepository.delete({});
+      await this.imageFileRepo.delete({});
+      await this.imageRepo.delete({});
     } catch (e) {
       return Fail(e);
     }

backend/src/collections/image-db/image-file-db.service.ts

@@ -0,0 +1,68 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { AsyncFailable, Fail } from 'picsur-shared/dist/types';
+import { Repository } from 'typeorm';
+import { ImageFileType } from '../../models/constants/image-file-types.const';
+import { EImageFileBackend } from '../../models/entities/image-file.entity';
+
+@Injectable()
+export class ImageFileDBService {
+  constructor(
+    @InjectRepository(EImageFileBackend)
+    private imageFileRepo: Repository<EImageFileBackend>,
+  ) {}
+
+  public async setSingle(
+    imageId: string,
+    type: ImageFileType,
+    file: Buffer,
+    mime: string,
+  ): AsyncFailable<true> {
+    const imageFile = new EImageFileBackend();
+    imageFile.imageId = imageId;
+    imageFile.type = type;
+    imageFile.mime = mime;
+    imageFile.data = file;
+
+    try {
+      await this.imageFileRepo.save(imageFile);
+    } catch (e) {
+      return Fail(e);
+    }
+
+    return true;
+  }
+
+  public async getSingle(
+    imageId: string,
+    type: ImageFileType,
+  ): AsyncFailable<EImageFileBackend> {
+    try {
+      const found = await this.imageFileRepo.findOne({
+        where: { imageId, type },
+      });
+
+      if (!found) return Fail('Image not found');
+      return found;
+    } catch (e) {
+      return Fail(e);
+    }
+  }
+
+  public async getSingleMime(
+    imageId: string,
+    type: ImageFileType,
+  ): AsyncFailable<string> {
+    try {
+      const found = await this.imageFileRepo.findOne({
+        where: { imageId, type },
+        select: ['mime'],
+      });
+
+      if (!found) return Fail('Image not found');
+      return found.mime;
+    } catch (e) {
+      return Fail(e);
+    }
+  }
+}

backend/src/collections/role-db/role-db.service.ts

@@ -58,15 +58,15 @@ export class RolesService {
   }
 
   public async getPermissions(roles: string[]): AsyncFailable<Permissions> {
-    const permissions: Permissions = [];
+    const foundRoles = await this.findMany(roles);
-    const foundRoles = await Promise.all(
+    if (HasFailed(foundRoles)) return foundRoles;
-      roles.map((role: string) => this.findOne(role)),
+
+    const permissions = foundRoles.reduce(
+      (acc, role) => [...acc, ...role.permissions],
+      [] as Permissions,
     );
 
-    for (const foundRole of foundRoles) {
+    console.log(permissions);
-      if (HasFailed(foundRole)) return foundRole;
-      permissions.push(...foundRole.permissions);
-    }
 
     return makeUnique(permissions);
   }
@@ -136,6 +136,19 @@ export class RolesService {
     }
   }
 
+  public async findMany(names: string[]): AsyncFailable<ERoleBackend[]> {
+    try {
+      const found = await this.rolesRepository.find({
+        where: { name: In(names) },
+      });
+
+      if (!found) return Fail('No roles found');
+      return found;
+    } catch (e) {
+      return Fail(e);
+    }
+  }
+
   public async findAll(): AsyncFailable<ERoleBackend[]> {
     try {
       const found = await this.rolesRepository.find();

backend/src/managers/image/image-processor.service.ts

@@ -11,6 +11,11 @@ import { QOIColorSpace, QOIdecode, QOIencode } from 'qoi-img';
 import sharp from 'sharp';
 import { UsrPreferenceService } from '../../collections/preference-db/usr-preference-db.service';
 
+interface ProcessResult {
+  image: Buffer;
+  mime: string;
+}
+
 @Injectable()
 export class ImageProcessorService {
   constructor(private readonly userPref: UsrPreferenceService) {}
@@ -19,7 +24,7 @@ export class ImageProcessorService {
     image: Buffer,
     mime: FullMime,
     userid: string,
-  ): AsyncFailable<Buffer> {
+  ): AsyncFailable<ProcessResult> {
     if (mime.type === SupportedMimeCategory.Image) {
       return await this.processStill(image, mime, {});
     } else if (mime.type === SupportedMimeCategory.Animation) {
@@ -27,25 +32,17 @@ export class ImageProcessorService {
     } else {
       return Fail('Unsupported mime type');
     }
-
-    // // nothing happens right now
-    // const keepOriginal = await this.userPref.getBooleanPreference(
-    //   userid,
-    //   UsrPreference.KeepOriginal,
-    // );
-    // if (HasFailed(keepOriginal)) return keepOriginal;
-
-    // if (keepOriginal) {
-    // }
   }
 
   private async processStill(
     image: Buffer,
     mime: FullMime,
     options: {},
-  ): AsyncFailable<Buffer> {
+  ): AsyncFailable<ProcessResult> {
+    let processedMime = mime.mime;
     let sharpImage: sharp.Sharp;
 
+    // TODO: ensure mime and sharp are in agreement
     if (mime.mime === ImageMime.ICO) {
       sharpImage = this.icoSharp(image);
     } else if (mime.mime === ImageMime.BMP) {
@@ -55,7 +52,7 @@ export class ImageProcessorService {
     } else {
       sharpImage = sharp(image);
     }
-    mime.mime = ImageMime.QOI;
+    processedMime = ImageMime.QOI;
 
     sharpImage = sharpImage.toColorspace('srgb');
 
@@ -69,6 +66,10 @@ export class ImageProcessorService {
     )
       return Fail('Invalid image');
 
+    if (metadata.width >= 32768 || metadata.height >= 32768) {
+      return Fail('Image too large');
+    }
+
     // Png can be more efficient than QOI, but its just sooooooo slow
     const qoiImage = QOIencode(pixels, {
       channels: metadata.hasAlpha ? 4 : 3,
@@ -77,16 +78,22 @@ export class ImageProcessorService {
       width: metadata.width,
     });
 
-    return qoiImage;
+    return {
+      image: qoiImage,
+      mime: processedMime,
+    };
   }
 
   private async processAnimation(
     image: Buffer,
     mime: FullMime,
     options: {},
-  ): AsyncFailable<Buffer> {
+  ): AsyncFailable<ProcessResult> {
     // Apng and gif are stored as is for now
-    return image;
+    return {
+      image: image,
+      mime: mime.mime,
+    };
   }
 
   private bmpSharp(image: Buffer) {

backend/src/managers/image/image.service.ts

@@ -5,6 +5,9 @@ import { AsyncFailable, HasFailed } from 'picsur-shared/dist/types';
 import { ParseMime } from 'picsur-shared/dist/util/parse-mime';
 import { IsQOI } from 'qoi-img';
 import { ImageDBService } from '../../collections/image-db/image-db.service';
+import { ImageFileDBService } from '../../collections/image-db/image-file-db.service';
+import { ImageFileType } from '../../models/constants/image-file-types.const';
+import { EImageFileBackend } from '../../models/entities/image-file.entity';
 import { EImageBackend } from '../../models/entities/image.entity';
 import { ImageProcessorService } from './image-processor.service';
 
@@ -16,6 +19,7 @@ import { ImageProcessorService } from './image-processor.service';
 export class ImageManagerService {
   constructor(
     private readonly imagesService: ImageDBService,
+    private readonly imageFilesService: ImageFileDBService,
     private readonly processService: ImageProcessorService,
   ) {}
 
@@ -25,10 +29,8 @@ export class ImageManagerService {
 
   // Image data buffer is not included by default, this also returns that buffer
   // Dont send to client, keep in backend
-  public async retrieveComplete(
+  public async retrieveComplete(id: string): AsyncFailable<EImageBackend> {
-    id: string,
+    return await this.imagesService.findOne(id);
-  ): AsyncFailable<Required<EImageBackend>> {
-    return await this.imagesService.findOne(id, true);
   }
 
   public async upload(
@@ -38,22 +40,69 @@ export class ImageManagerService {
     const fullMime = await this.getFullMimeFromBuffer(image);
     if (HasFailed(fullMime)) return fullMime;
 
-    const processedImage = await this.processService.process(
+    const processResult = await this.processService.process(
       image,
       fullMime,
       userid,
     );
-    if (HasFailed(processedImage)) return processedImage;
+    if (HasFailed(processResult)) return processResult;
 
-    const imageEntity = await this.imagesService.create(
+    const imageEntity = await this.imagesService.create();
-      processedImage,
-      fullMime.mime,
-    );
     if (HasFailed(imageEntity)) return imageEntity;
 
+    const imageFileEntity = await this.imageFilesService.setSingle(
+      imageEntity.id,
+      ImageFileType.MASTER,
+      processResult.image,
+      processResult.mime,
+    );
+    if (HasFailed(imageFileEntity)) return imageFileEntity;
+
+    // // nothing happens right now
+    // const keepOriginal = await this.userPref.getBooleanPreference(
+    //   userid,
+    //   UsrPreference.KeepOriginal,
+    // );
+    // if (HasFailed(keepOriginal)) return keepOriginal;
+
+    // if (keepOriginal) {
+    // }
+
     return imageEntity;
   }
 
+  // File getters ==============================================================
+
+  public async getMaster(imageId: string): AsyncFailable<EImageFileBackend> {
+    return this.imageFilesService.getSingle(imageId, ImageFileType.MASTER);
+  }
+
+  public async getMasterMime(imageId: string): AsyncFailable<FullMime> {
+    const mime = await this.imageFilesService.getSingleMime(
+      imageId,
+      ImageFileType.MASTER,
+    );
+    if (HasFailed(mime)) return mime;
+
+    return ParseMime(mime);
+  }
+
+  public async getOriginal(imageId: string): AsyncFailable<EImageFileBackend> {
+    return this.imageFilesService.getSingle(imageId, ImageFileType.ORIGINAL);
+  }
+
+  public async getOriginalMime(imageId: string): AsyncFailable<FullMime> {
+    const mime = await this.imageFilesService.getSingleMime(
+      imageId,
+      ImageFileType.ORIGINAL,
+    );
+    if (HasFailed(mime)) return mime;
+
+    return ParseMime(mime);
+  }
+
+  // Util stuff ==================================================================
+
   private async getFullMimeFromBuffer(image: Buffer): AsyncFailable<FullMime> {
     const filetypeResult: FileTypeResult | undefined = await fileTypeFromBuffer(
       image,

backend/src/models/constants/image-file-types.const.ts

@@ -0,0 +1,5 @@
+export enum ImageFileType {
+  ORIGINAL = 'original',
+  MASTER = 'master',
+  DERIVED = 'derived',
+}

backend/src/models/entities/image-file.entity.ts

@@ -0,0 +1,23 @@
+import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
+import { ImageFileType } from '../constants/image-file-types.const';
+
+@Entity()
+export class EImageFileBackend {
+  @PrimaryGeneratedColumn('uuid')
+  private _id?: string;
+
+  @Column({ nullable: false })
+  @Index()
+  imageId: string;
+
+  @Column({ nullable: false, enum: ImageFileType })
+  @Index()
+  type: ImageFileType;
+
+  @Column({ nullable: false })
+  mime: string;
+
+  // Binary data
+  @Column({ type: 'bytea', nullable: false })
+  data: Buffer;
+}

backend/src/models/entities/image.entity.ts

@@ -1,26 +1,8 @@
-import { EImageSchema } from 'picsur-shared/dist/entities/image.entity';
+import { EImage } from 'picsur-shared/dist/entities/image.entity';
-import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';
+import { Entity, PrimaryGeneratedColumn } from 'typeorm';
-import { z } from 'zod';
-
-const OverriddenEImageSchema = EImageSchema.omit({ data: true }).merge(
-  z.object({
-    data: z.any(),
-  }),
-);
-type OverriddenEImage = z.infer<typeof OverriddenEImageSchema>;
 
 @Entity()
-export class EImageBackend implements OverriddenEImage {
+export class EImageBackend implements EImage {
   @PrimaryGeneratedColumn('uuid')
   id: string;
-
-  @Column({ nullable: false })
-  mime: string;
-
-  // Binary data
-  @Column({ type: 'bytea', nullable: false, select: false })
-  data?: Buffer;
-
-  @Column({ type: 'bytea', nullable: true, select: false })
-  originaldata?: Buffer;
 }

backend/src/models/entities/index.ts

@@ -1,3 +1,4 @@
+import { EImageFileBackend } from './image-file.entity';
 import { EImageBackend } from './image.entity';
 import { ERoleBackend } from './role.entity';
 import { ESysPreferenceBackend } from './sys-preference.entity';
@@ -6,6 +7,7 @@ import { EUsrPreferenceBackend } from './usr-preference.entity';
 
 export const EntityList = [
   EImageBackend,
+  EImageFileBackend,
   EUserBackend,
   ERoleBackend,
   ESysPreferenceBackend,

backend/src/models/transformers/image.transformer.ts

@@ -1,14 +0,0 @@
-import { EImage } from 'picsur-shared/dist/entities/image.entity';
-import { EImageBackend } from '../entities/image.entity';
-
-export function EImageBackend2EImage(
-  eImage: EImageBackend,
-): EImage {
-  if (eImage.data === undefined) 
-    return eImage as EImage;
-
-  return {
-    ...eImage,
-    data: undefined,
-  };
-}

backend/src/routes/image/image.controller.ts

@@ -19,7 +19,6 @@ import { Returns } from '../../decorators/returns.decorator';
 import { ImageManagerService } from '../../managers/image/image.service';
 import { Permission } from '../../models/constants/permissions.const';
 import { ImageUploadDto } from '../../models/dto/image-upload.dto';
-import { EImageBackend2EImage } from '../../models/transformers/image.transformer';
 
 // This is the only controller with CORS enabled
 @Controller('i')
@@ -36,7 +35,7 @@ export class ImageController {
     @Res({ passthrough: true }) res: FastifyReply,
     @ImageIdParam() id: string,
   ): Promise<Buffer> {
-    const image = await this.imagesService.retrieveComplete(id);
+    const image = await this.imagesService.getMaster(id);
     if (HasFailed(image)) {
       this.logger.warn(image.getReason());
       throw new NotFoundException('Could not find image');
@@ -51,13 +50,13 @@ export class ImageController {
     @Res({ passthrough: true }) res: FastifyReply,
     @ImageIdParam() id: string,
   ) {
-    const image = await this.imagesService.retrieveInfo(id);
+    const fullmime = await this.imagesService.getMasterMime(id);
-    if (HasFailed(image)) {
+    if (HasFailed(fullmime)) {
-      this.logger.warn(image.getReason());
+      this.logger.warn(fullmime.getReason());
       throw new NotFoundException('Could not find image');
     }
 
-    res.type(image.mime);
+    res.type(fullmime.mime);
   }
 
   @Get('meta/:id')
@@ -69,7 +68,7 @@ export class ImageController {
       throw new NotFoundException('Could not find image');
     }
 
-    return EImageBackend2EImage(image);
+    return image;
   }
 
   @Post()
@@ -79,12 +78,15 @@ export class ImageController {
     @MultiPart() multipart: ImageUploadDto,
     @ReqUserID() userid: string,
   ): Promise<ImageMetaResponse> {
-    const image = await this.imagesService.upload(multipart.image.buffer, userid);
+    const image = await this.imagesService.upload(
+      multipart.image.buffer,
+      userid,
+    );
     if (HasFailed(image)) {
-      this.logger.warn(image.getReason());
+      this.logger.warn(image.getReason(), image.getStack());
       throw new InternalServerErrorException('Could not upload image');
     }
 
-    return EImageBackend2EImage(image);
+    return image;
   }
 }

shared/src/entities/image.entity.ts

@@ -3,7 +3,5 @@ import { IsEntityID } from '../validators/entity-id.validator';
 
 export const EImageSchema = z.object({
   id: IsEntityID(),
-  data: z.undefined(),
-  mime: z.string(),
 });
 export type EImage = z.infer<typeof EImageSchema>;