From f470581189df996549b8e36248878f36945a7d55 Mon Sep 17 00:00:00 2001 From: Caramel Date: Fri, 8 Nov 2024 16:06:54 +0100 Subject: [PATCH] Add support for heif, jpeg-xl and jpeg2000 --- .gitignore | 2 ++ README.md | 2 +- backend/package.json | 3 +- backend/src/workers/sharp/sharp.worker.ts | 12 +++++--- backend/src/workers/sharp/universal-sharp.ts | 20 +++++++++++++ frontend/package.json | 2 +- pnpm-lock.yaml | 12 +++++++- shared/package.json | 2 +- shared/src/dto/mimes.dto.ts | 12 ++++++++ support/dev.docker-compose.yml | 2 +- support/picsur-stage2.Dockerfile | 30 +++++++++++++++++--- support/picsur.docker-compose.yml | 4 +-- 12 files changed, 87 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 3210a00..ddbe7ed 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ yarn-error.log .pnp.* temp + +**/.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index b15efec..7671a0a 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ services: # PICSUR_VERBOSE: "true" restart: unless-stopped picsur_postgres: - image: postgres:14-alpine + image: postgres:17-alpine container_name: picsur_postgres environment: POSTGRES_DB: picsur diff --git a/backend/package.json b/backend/package.json index 3fb85ab..9a850cd 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "picsur-backend", - "version": "0.5.6", + "version": "0.5.7", "description": "Backend for Picsur", "license": "GPL-3.0", "repository": "https://github.com/caramelfur/Picsur", @@ -42,6 +42,7 @@ "file-type": "^19.6.0", "is-docker": "^3.0.0", "ms": "2.1.3", + "node-addon-api": "^8.2.2", "node-fetch": "^3.3.2", "p-timeout": "^6.1.3", "passport": "^0.7.0", diff --git a/backend/src/workers/sharp/sharp.worker.ts b/backend/src/workers/sharp/sharp.worker.ts index c0a3078..765d5e0 100644 --- a/backend/src/workers/sharp/sharp.worker.ts +++ b/backend/src/workers/sharp/sharp.worker.ts @@ -29,10 +29,14 @@ export class SharpWorker { return this.purge('MEMORY_LIMIT_MB environment variable is not set'); } - setrlimit('data', { - soft: 1000 * 1000 * memoryLimit, - hard: 1000 * 1000 * memoryLimit, - }); + try { + setrlimit('data', { + soft: 1000 * 1000 * memoryLimit, + hard: 1000 * 1000 * memoryLimit, + }); + } catch (e) { + console.warn('Failed to set memory limit'); + } process.on('message', this.messageHandler.bind(this)); diff --git a/backend/src/workers/sharp/universal-sharp.ts b/backend/src/workers/sharp/universal-sharp.ts index 097147e..7f7a3af 100644 --- a/backend/src/workers/sharp/universal-sharp.ts +++ b/backend/src/workers/sharp/universal-sharp.ts @@ -93,6 +93,26 @@ export async function UniversalSharpOut( .tiff({ quality: options?.quality }) .toBuffer({ resolveWithObject: true }); break; + case ImageFileType.AVIF: + result = await image + .avif({ quality: options?.quality }) + .toBuffer({ resolveWithObject: true }); + break; + case ImageFileType.HEIF: + result = await image + .heif({ quality: options?.quality, compression: 'av1' }) + .toBuffer({ resolveWithObject: true }); + break; + case ImageFileType.JXL: + result = await image + .jxl({ quality: options?.quality }) + .toBuffer({ resolveWithObject: true }); + break; + case ImageFileType.JP2: + result = await image + .jp2({ quality: options?.quality }) + .toBuffer({ resolveWithObject: true }); + break; case ImageFileType.BMP: result = await bmpSharpOut(image); break; diff --git a/frontend/package.json b/frontend/package.json index 35982ef..4a339af 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "picsur-frontend", - "version": "0.5.6", + "version": "0.5.7", "description": "Frontend for Picsur", "license": "GPL-3.0", "repository": "https://github.com/caramelfur/Picsur", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 480efd1..54beeea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,9 @@ importers: ms: specifier: 2.1.3 version: 2.1.3 + node-addon-api: + specifier: ^8.2.2 + version: 8.2.2 node-fetch: specifier: ^3.3.2 version: 3.3.2 @@ -2057,7 +2060,7 @@ packages: peerDependencies: '@angular/compiler-cli': ^18.0.0 typescript: ~5.5.4 - webpack: '>=5.76.0' + webpack: ^5.54.0 '@ngui/common@1.0.0': resolution: {integrity: sha512-T0vX6jFLR+19iUVqM0J6lQkcDo6Iaq8pptzMJDEjLG3HkpgeM9SYxiTFV3+yHuP4QzQQ6/VP8gJ+1f4M7iZv5Q==} @@ -3113,6 +3116,7 @@ packages: critters@0.0.24: resolution: {integrity: sha512-Oyqew0FGM0wYUSNqR0L6AteO5MpMoUU0rhKRieXeiKs+PmRTxiJMyaunYB2KF6fQ3dzChXKCpbFOEJx3OQ1v/Q==} + deprecated: Ownership of Critters has moved to the Nuxt team, who will be maintaining the project going forward. If you'd like to keep using Critters, please switch to the actively-maintained fork at https://github.com/danielroe/beasties cron@3.1.7: resolution: {integrity: sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==} @@ -4449,6 +4453,10 @@ packages: node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-addon-api@8.2.2: + resolution: {integrity: sha512-9emqXAKhVoNrQ792nLI/wpzPpJ/bj/YXxW0CvAau1+RdGBcCRF1Dmz7719zgVsQNrzHl9Tzn3ImZ4qWFarWL0A==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -10437,6 +10445,8 @@ snapshots: node-addon-api@7.1.1: {} + node-addon-api@8.2.2: {} + node-domexception@1.0.0: {} node-emoji@1.11.0: diff --git a/shared/package.json b/shared/package.json index 876e999..26e7d38 100644 --- a/shared/package.json +++ b/shared/package.json @@ -1,6 +1,6 @@ { "name": "picsur-shared", - "version": "0.5.6", + "version": "0.5.7", "description": "Shared libraries for Picsur", "license": "GPL-3.0", "repository": "https://github.com/caramelfur/Picsur", diff --git a/shared/src/dto/mimes.dto.ts b/shared/src/dto/mimes.dto.ts index 01ae269..f9d1384 100644 --- a/shared/src/dto/mimes.dto.ts +++ b/shared/src/dto/mimes.dto.ts @@ -7,6 +7,10 @@ export enum ImageFileType { PNG = 'image:png', WEBP = 'image:webp', TIFF = 'image:tiff', + AVIF = 'image:avif', + HEIF = 'image:heif', + JXL = 'image:jxl', + JP2 = 'image:jp2', BMP = 'image:bmp', // ICO = 'image:ico', } @@ -51,6 +55,10 @@ const FileType2MimeMap: { [ImageFileType.PNG]: 'image/png', [ImageFileType.WEBP]: 'image/webp', // Image webp comes later, so will be default [ImageFileType.TIFF]: 'image/tiff', + [ImageFileType.AVIF]: 'image/avif', + [ImageFileType.HEIF]: 'image/heic', + [ImageFileType.JXL]: 'image/jxl', + [ImageFileType.JP2]: 'image/jp2', [ImageFileType.BMP]: 'image/bmp', // [ImageFileType.ICO]: 'image/x-icon', }; @@ -83,6 +91,10 @@ const FileType2ExtMap: { [ImageFileType.PNG]: 'png', [ImageFileType.WEBP]: 'webp', [ImageFileType.TIFF]: 'tiff', + [ImageFileType.AVIF]: 'avif', + [ImageFileType.HEIF]: 'heif', + [ImageFileType.JXL]: 'jxl', + [ImageFileType.JP2]: 'jp2', [ImageFileType.BMP]: 'bmp', // [ImageFileType.ICO]: 'ico', }; diff --git a/support/dev.docker-compose.yml b/support/dev.docker-compose.yml index 2d7e33c..ce03f95 100644 --- a/support/dev.docker-compose.yml +++ b/support/dev.docker-compose.yml @@ -1,6 +1,6 @@ services: devdb: - image: postgres:14-alpine + image: postgres:17-alpine environment: POSTGRES_DB: picsur POSTGRES_PASSWORD: picsur diff --git a/support/picsur-stage2.Dockerfile b/support/picsur-stage2.Dockerfile index 98f7181..5ba1f8a 100644 --- a/support/picsur-stage2.Dockerfile +++ b/support/picsur-stage2.Dockerfile @@ -3,18 +3,40 @@ # Always fetch amd64 image FROM ghcr.io/caramelfur/picsur-alpha-stage1:latest AS builder_stage1 -FROM node:20-alpine AS builder_stage2 +# === START VIPS === +FROM node:20-alpine AS vips_builder +RUN apk add build-base meson cmake pkgconfig glib-dev \ + expat-dev libexif-dev jpeg-dev libjxl-dev openjpeg-dev libpng-dev \ + tiff-dev libheif-dev libwebp-dev cgif-dev imagemagick-dev \ + librsvg-dev pango-dev cfitsio-dev poppler-dev libspng-dev \ + lcms2-dev fftw-dev libimagequant-dev +WORKDIR /vips +RUN wget https://github.com/libvips/libvips/releases/download/v8.16.0/vips-8.16.0.tar.xz -O vips.tar.xz +RUN tar -xf vips.tar.xz && \ + cd vips-* && \ + meson setup build -Dexamples=false && \ + cd build && \ + meson compile && \ + meson install -RUN npm install -g pnpm +FROM node:20-alpine AS vips_clean +COPY --from=vips_builder /usr/local /usr/local +RUN apk add pkgconfig glib expat libexif jpeg libjxl openjpeg libpng tiff \ + libheif libwebp cgif imagemagick librsvg pango cfitsio poppler \ + libspng lcms2 fftw libimagequant +# === END VIPS === +FROM vips_builder AS builder_stage2 + +RUN npm install -g pnpm node-gyp RUN apk add python3 build-base WORKDIR /picsur COPY --from=builder_stage1 /picsur ./ - +ENV SHARP_FORCE_GLOBAL_LIBVIPS=1 RUN pnpm install --frozen-lockfile --prod -FROM node:20-alpine +FROM vips_clean RUN npm install -g pnpm diff --git a/support/picsur.docker-compose.yml b/support/picsur.docker-compose.yml index ae3ed38..85ed27b 100644 --- a/support/picsur.docker-compose.yml +++ b/support/picsur.docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: picsur: - image: ghcr.io/caramelfur/picsur:latest + image: ghcr.io/caramelfur/picsur-alpha:latest container_name: picsur ports: - '8080:8080' @@ -30,7 +30,7 @@ services: # PICSUR_VERBOSE: "true" restart: unless-stopped picsur_postgres: - image: postgres:14-alpine + image: postgres:17-alpine container_name: picsur_postgres environment: POSTGRES_DB: picsur