From dc5bcbe9b26fade35a6e73eb285c942f9ef85c7f Mon Sep 17 00:00:00 2001 From: Meier Lukas Date: Sat, 10 Jun 2023 11:07:49 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Migrate=20rss=20to=20tR?= =?UTF-8?q?PC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/api/root.ts | 2 + src/server/api/routers/rss.ts | 185 ++++++++++++++++++++++++++++++ src/widgets/rss/RssWidgetTile.tsx | 83 ++++++++------ 3 files changed, 236 insertions(+), 34 deletions(-) create mode 100644 src/server/api/routers/rss.ts diff --git a/src/server/api/root.ts b/src/server/api/root.ts index cf28aaa48..2f8a5e08b 100644 --- a/src/server/api/root.ts +++ b/src/server/api/root.ts @@ -1,5 +1,6 @@ import { createTRPCRouter } from '~/server/api/trpc'; import { appRouter } from './routers/app'; +import { rssRouter } from './routers/rss'; /** * This is the primary router for your server. @@ -8,6 +9,7 @@ import { appRouter } from './routers/app'; */ export const rootRouter = createTRPCRouter({ app: appRouter, + rss: rssRouter, }); // export type definition of API diff --git a/src/server/api/routers/rss.ts b/src/server/api/routers/rss.ts new file mode 100644 index 000000000..e60c2dd1a --- /dev/null +++ b/src/server/api/routers/rss.ts @@ -0,0 +1,185 @@ +import { z } from 'zod'; +import RssParser from 'rss-parser'; +import Consola from 'consola'; +import { decode, encode } from 'html-entities'; +import xss from 'xss'; +import { TRPCError } from '@trpc/server'; +import { createTRPCRouter, publicProcedure } from '../trpc'; +import { Stopwatch } from '~/tools/shared/time/stopwatch.tool'; +import { getConfig } from '~/tools/config/getConfig'; +import { IRssWidget } from '~/widgets/rss/RssWidgetTile'; + +type CustomItem = { + 'media:content': string; + enclosure: { + url: string; + }; +}; + +const rssFeedResultObjectSchema = z + .object({ + success: z.literal(false), + feed: z.undefined(), + }) + .or( + z.object({ + success: z.literal(true), + feed: z.object({ + title: z.string().or(z.undefined()), + items: z.array( + z.object({ + link: z.string(), + enclosure: z + .object({ + url: z.string(), + }) + .or(z.undefined()), + categories: z.array(z.string()).or(z.undefined()), + title: z.string(), + content: z.string(), + pubDate: z.string(), + }) + ), + }), + }) + ); + +export const rssRouter = createTRPCRouter({ + all: publicProcedure + .input( + z.object({ + widgetId: z.string().uuid(), + feedUrls: z.array(z.string()), + configName: z.string(), + }) + ) + .output(z.array(rssFeedResultObjectSchema)) + .query(async ({ input }) => { + const config = getConfig(input.configName); + + const rssWidget = config.widgets.find((x) => x.type === 'rss' && x.id === input.widgetId) as + | IRssWidget + | undefined; + + if (!rssWidget || input.feedUrls.length === 0) { + throw new TRPCError({ + code: 'NOT_FOUND', + message: 'required widget does not exist', + }); + } + + const result = await Promise.all( + input.feedUrls.map(async (feedUrl) => + getFeedUrl(feedUrl, rssWidget.properties.dangerousAllowSanitizedItemContent) + ) + ); + return result; + }), +}); + +const getFeedUrl = async (feedUrl: string, dangerousAllowSanitizedItemContent: boolean) => { + Consola.info(`Requesting RSS feed at url ${feedUrl}`); + const stopWatch = new Stopwatch(); + const feed = await parser.parseURL(feedUrl); + Consola.info(`Retrieved RSS feed after ${stopWatch.getEllapsedMilliseconds()} milliseconds`); + + const orderedFeed = { + ...feed, + items: feed.items + .map( + (item: { + title: string; + content: string; + 'content:encoded': string; + categories: string[] | { _: string }[]; + }) => ({ + ...item, + categories: item.categories + ?.map((category) => (typeof category === 'string' ? category : category._)) + .filter((category: unknown): category is string => typeof category === 'string'), + title: item.title ? decode(item.title) : undefined, + content: processItemContent( + item['content:encoded'] ?? item.content, + dangerousAllowSanitizedItemContent + ), + enclosure: createEnclosure(item), + link: createLink(item), + }) + ) + .sort((a: { pubDate: number }, b: { pubDate: number }) => { + if (!a.pubDate || !b.pubDate) { + return 0; + } + + return a.pubDate - b.pubDate; + }) + .slice(0, 20), + }; + + return { + feed: orderedFeed, + success: orderedFeed?.items !== undefined, + }; +}; + +const processItemContent = (content: string, dangerousAllowSanitizedItemContent: boolean) => { + if (dangerousAllowSanitizedItemContent) { + return xss(content, { + allowList: { + p: [], + h1: [], + h2: [], + h3: [], + h4: [], + h5: [], + h6: [], + a: ['href'], + b: [], + strong: [], + i: [], + em: [], + img: ['src', 'width', 'height'], + br: [], + small: [], + ul: [], + li: [], + ol: [], + figure: [], + svg: [], + code: [], + mark: [], + blockquote: [], + }, + }); + } + + return encode(content); +}; + +const createLink = (item: any) => { + if (item.link) { + return item.link; + } + + return item.guid; +}; + +const createEnclosure = (item: any) => { + if (item.enclosure) { + return item.enclosure; + } + + if (item['media:content']) { + return { + url: item['media:content'].$.url, + }; + } + + return undefined; +}; + +const parser: RssParser = new RssParser({ + customFields: { + item: ['media:content', 'enclosure'], + }, +}); diff --git a/src/widgets/rss/RssWidgetTile.tsx b/src/widgets/rss/RssWidgetTile.tsx index dc1cebe68..9512bc2ef 100644 --- a/src/widgets/rss/RssWidgetTile.tsx +++ b/src/widgets/rss/RssWidgetTile.tsx @@ -15,11 +15,12 @@ import { createStyles, } from '@mantine/core'; import { IconClock, IconRefresh, IconRss } from '@tabler/icons-react'; -import { useQuery } from '@tanstack/react-query'; import dayjs from 'dayjs'; import { useTranslation } from 'next-i18next'; import Link from 'next/link'; +import { useConfigContext } from '~/config/provider'; +import { api } from '~/utils/api'; import { defineWidget } from '../helper'; import { IWidget } from '../widgets'; @@ -65,27 +66,11 @@ interface RssTileProps { widget: IRssWidget; } -export const useGetRssFeeds = (feedUrls: string[], refreshInterval: number, widgetId: string) => - useQuery({ - queryKey: ['rss-feeds', feedUrls], - // Cache the results for 24 hours - cacheTime: 1000 * 60 * 60 * 24, - staleTime: 1000 * 60 * refreshInterval, - queryFn: async () => { - const responses = await Promise.all( - feedUrls.map((feedUrl) => - fetch( - `/api/modules/rss?widgetId=${widgetId}&feedUrl=${encodeURIComponent(feedUrl)}` - ).then((response) => response.json()) - ) - ); - return responses; - }, - }); - function RssTile({ widget }: RssTileProps) { const { t } = useTranslation('modules/rss'); + const { name: configName } = useConfigContext(); const { data, isLoading, isFetching, isError, refetch } = useGetRssFeeds( + configName, widget.properties.rssFeedUrl, widget.properties.refreshInterval, widget.id @@ -122,6 +107,7 @@ function RssTile({ widget }: RssTileProps) { {t('descriptor.card.errors.general.title')} {t('descriptor.card.errors.general.text')} + ); } @@ -192,25 +178,54 @@ function RssTile({ widget }: RssTileProps) { ))} - refetch()} - bottom={10} - styles={{ - root: { - borderColor: 'red', - }, - }} - > - {isFetching ? : } - + ); } +export const useGetRssFeeds = ( + configName: string | undefined, + feedUrls: string[], + refreshInterval: number, + widgetId: string +) => + api.rss.all.useQuery( + { + configName: configName ?? '', + feedUrls, + widgetId, + }, + { + // Cache the results for 24 hours + cacheTime: 1000 * 60 * 60 * 24, + staleTime: 1000 * 60 * refreshInterval, + enabled: !!configName, + } + ); + +interface RefetchButtonProps { + refetch: () => void; + isFetching: boolean; +} + +const RefetchButton = ({ isFetching, refetch }: RefetchButtonProps) => ( + refetch()} + bottom={10} + styles={{ + root: { + borderColor: 'red', + }, + }} + > + {isFetching ? : } + +); + const InfoDisplay = ({ date, title }: { date: string; title: string | undefined }) => (