From 81729388fc3d89212c15ed642a25c60d32758b14 Mon Sep 17 00:00:00 2001 From: Tarrgon <61888458+Tarrgon@users.noreply.github.com> Date: Sun, 14 Jun 2026 15:22:50 -0400 Subject: [PATCH] Fix markdown parsing and blacklisted appeals --- package-lock.json | 8 +- package.json | 2 +- src/events/handle-message.ts | 1 - src/utils/appeal-events.ts | 31 +++++--- src/utils/dtext-utils.ts | 96 ++++++++++++++++++++++++ src/utils/e621-utils.ts | 6 ++ src/utils/event-utils.ts | 141 ++--------------------------------- src/utils/index.ts | 1 + src/utils/ticket-events.ts | 12 ++- 9 files changed, 142 insertions(+), 156 deletions(-) create mode 100644 src/utils/dtext-utils.ts diff --git a/package-lock.json b/package-lock.json index e9ddc33..5cffe3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "@clynamic/dmark": "0.0.8", + "@clynamic/dmark": "0.0.11", "@redis/client": "5.1.0", "body-parser": "2.2.2", "discord.js": "14.26.4", @@ -31,9 +31,9 @@ } }, "node_modules/@clynamic/dmark": { - "version": "0.0.8", - "resolved": "https://npm.clynamic.net/%40clynamic%2Fdmark/-/0.0.8/dmark-0.0.8.tgz", - "integrity": "sha512-KzZqTeWldrB2ij74s5hVUVdHaBur4QqYExv6V9mSiTovdcspbRS0pA5v6OwONdxvcBou7b4p4cIalNj4eARgxg==", + "version": "0.0.11", + "resolved": "https://npm.clynamic.net/%40clynamic%2Fdmark/-/0.0.11/dmark-0.0.11.tgz", + "integrity": "sha512-efrxWJ6ONQ6kcCLwPfoIonrMznvYYm9fZ1v+DgDY5BzQ8EDrgbCJr3ri9dVS7VUt9Rh8yZTXtaMDI1jf3Gb0Aw==", "license": "UNLICENSED" }, "node_modules/@discordjs/builders": { diff --git a/package.json b/package.json index f96c9ed..79443a9 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "typescript-eslint": "8.32.1" }, "dependencies": { - "@clynamic/dmark": "0.0.8", + "@clynamic/dmark": "0.0.11", "@redis/client": "5.1.0", "body-parser": "2.2.2", "discord.js": "14.26.4", diff --git a/src/events/handle-message.ts b/src/events/handle-message.ts index d5a633f..def9990 100644 --- a/src/events/handle-message.ts +++ b/src/events/handle-message.ts @@ -50,7 +50,6 @@ const regexTesters = [ const uniqueRegexMatches = (g, i, a) => a.findIndex(v => v[1] == g[1]) == i; export async function handleMessageCreate(message: Message) { - console.log('message created'); if (message.author.bot) return; if (message.inGuild()) await Database.putMessage(message); diff --git a/src/utils/appeal-events.ts b/src/utils/appeal-events.ts index 7292787..04f9252 100644 --- a/src/utils/appeal-events.ts +++ b/src/utils/appeal-events.ts @@ -2,10 +2,10 @@ import { ActionRowBuilder, APIEmbedField, ButtonBuilder, ButtonStyle, Client, Di import { config } from '../config'; import { Database } from '../shared/Database'; import { Appeal, AppealUpdate, E621Post, PostFlag } from '../types'; -import { getAuthor, getColor, getDescription, getFields, getLinks } from './event-utils'; +import { getAuthor, getColor, parseMarkdownToField, getFields } from './event-utils'; import { humanizeCapitalization } from './string-utils'; -import { getE621Post, getE621PostFlag } from './e621-utils'; -import { parseDTextToMarkdown } from '@clynamic/dmark'; +import { getE621Post, getE621PostFlag, PostAction, spoilerOrBlacklist } from './e621-utils'; +import { parseDTextToMarkdown } from './dtext-utils'; export async function appealUpdateHandler(client: Client, update: string) { const data: AppealUpdate = JSON.parse(update); @@ -92,7 +92,7 @@ async function getCustomFields(appeal: Appeal, flag: PostFlag, post: E621Post): return [ { name: 'Deletion Reason', - value: parseDTextToMarkdown(await getLinks(flag.reason)).output, + value: await parseDTextToMarkdown(flag.reason), inline: true } ]; @@ -102,22 +102,31 @@ async function createEmbedFromAppeal(appeal: Appeal, flag: PostFlag, post: E621P return new EmbedBuilder() .setTitle(getTitle(appeal)) .setURL(await getURL(appeal)) - .setDescription(await getDescription(appeal)) .setAuthor(getAuthor(appeal)) .setColor(getColor(appeal)) - .setFields(...getFields(appeal), ...(await getCustomFields(appeal, flag, post))) + .setFields( + { + name: 'Reason', + value: await parseMarkdownToField(appeal.reason), + inline: false + }, + ...getFields(appeal), + ...(await getCustomFields(appeal, flag, post)) + ) .setFooter({ text: `Appeal #${appeal.id}` }); } async function getButtons(appeal: Appeal, flag: PostFlag, post: E621Post): Promise> { const row = new ActionRowBuilder(); - const button = new ButtonBuilder() - .setLabel('Open Post') - .setStyle(ButtonStyle.Link) - .setURL(`${config.E621_BASE_URL}/posts/${post.id}`); + if (spoilerOrBlacklist(post).action != PostAction.Blacklist) { + const button = new ButtonBuilder() + .setLabel('Open Post') + .setStyle(ButtonStyle.Link) + .setURL(`${config.E621_BASE_URL}/posts/${post.id}`); - row.addComponents(button); + row.addComponents(button); + } return row; } \ No newline at end of file diff --git a/src/utils/dtext-utils.ts b/src/utils/dtext-utils.ts new file mode 100644 index 0000000..35a8c1f --- /dev/null +++ b/src/utils/dtext-utils.ts @@ -0,0 +1,96 @@ +import { BlockNode, DocumentNode, formatMarkdown, InlineNode, LinkNode, MarkdownFormatContext, MarkdownHandlers, markdownHandlers, parseDTextToAST, TableBodyNode, TableHeadNode, TableLiteralNode, TableRowNode, TextNode } from '@clynamic/dmark'; +import { config } from '../config'; +import { E621Post } from '../types'; +import { getManyE621Posts, PostAction, SEARCH_LIMIT, spoilerOrBlacklist } from './e621-utils'; + +export async function parseDTextToMarkdown(text: string): Promise { + const ast = parseDTextToAST(text, { + baseUrl: config.E621_BASE_URL, + allowColor: false + }) as DocumentNode; + + const postIds: string[][] = []; + + const recurseChildren = (node: BlockNode | InlineNode | TableHeadNode | TableBodyNode | TableRowNode | TableLiteralNode | LinkNode) => { + + if (node.type == 'link' && node.linkType == 'id_link' && node.id) { + if (postIds.length == 0 || postIds.at(-1)!.length >= SEARCH_LIMIT) postIds.push([]); + postIds.at(-1)!.push(node.id); + } + + if ('children' in node && node.children !== undefined) { + for (const child of node.children) recurseChildren(child); + } + }; + + for (const child of ast.children) recurseChildren(child); + + const postData: E621Post[] = []; + + for (const chunk of postIds) { + postData.push(...await getManyE621Posts(chunk)); + } + + const handlers: MarkdownHandlers = { + ...markdownHandlers, + + link: (node: LinkNode, out: string[], ctx: MarkdownFormatContext) => { + ctx.atLineStart = false; + if (node.linkType == 'wiki') { + formatWikiLink(node, out); + } else if (node.linkType == 'id_link' && node.idType == 'post') { + const post = postData.find(p => p.id == Number(node.id)); + if (post && spoilerOrBlacklist(post).action == PostAction.Blacklist) { + out.push('post #', node.id!); + } else { + out.push('['); + out.push('post #', node.id!); + out.push(']'); + out.push(`(${node.href.startsWith('/') ? config.E621_BASE_URL : ''}${node.href})`); + } + } else { + out.push('['); + markdownHandlers.link(node, out, ctx); + out.push(']'); + out.push(`(${node.href.startsWith('/') ? config.E621_BASE_URL : ''}${node.href})`); + } + } + }; + + return formatMarkdown(ast, {}, handlers).output; +} + + +// Modified from +// https://github.com/clragon/dmark/blob/main/src/ast/text.ts#L17-L21 +// https://github.com/clragon/dmark/blob/main/src/md/render/index.ts#L481-L531 +const RE_HAS_UPPER = /[A-Z]/; +export function asciiLowercase(s: string): string { + if (!RE_HAS_UPPER.test(s)) return s; + return s.replace(/[A-Z]/g, c => String.fromCharCode(c.charCodeAt(0) + 32)); +} + +function formatWikiLink(node: LinkNode, out: string[]): void { + // Same dispatch as the dtext sibling formatter (ADR-0004). Anchor-only + // form has two variants (`[[#anchor]]` and `[[#anchor|title]]`); detect + // title-override by comparing children content to the default form. + if (node.href.startsWith('#') && node.anchor !== undefined) { + const childText + = node.children?.[0] && node.children[0].type === 'text' + ? (node.children[0] as TextNode).content + : ''; + if (childText === `#${node.anchor}` || childText === '') { + out.push('[[#', node.anchor, ']]'); + } else { + out.push('[[#', childText, ']]'); + } + return; + } + + const childText + = node.children?.[0] && node.children[0].type === 'text' + ? (node.children[0] as TextNode).content + : ''; + + out.push('[', '[[', childText, ']]', ']', `(${config.E621_BASE_URL}${node.href})`); +} \ No newline at end of file diff --git a/src/utils/e621-utils.ts b/src/utils/e621-utils.ts index 4209680..474502b 100644 --- a/src/utils/e621-utils.ts +++ b/src/utils/e621-utils.ts @@ -9,6 +9,8 @@ const SPOILERED_NONSAFE_TAGS: string[] = []; const USER_AGENT = 'E621DiscordBot'; +export const SEARCH_LIMIT = 320; + async function request(path: string, query?: { [name: string]: string }): Promise { const url = new URL(config.E621_BASE_URL!); url.pathname = path + '.json'; @@ -38,6 +40,10 @@ export async function getE621Post(id: string | number): Promise return (await request(`/posts/${id}`, { v2: 'true' })) as E621Post ?? null; } +export async function getManyE621Posts(ids: string[] | number[]): Promise { + return (await request('/posts', { v2: 'true', tags: `id:${ids.join(',')}` })) as E621Post[] ?? []; +} + export async function getE621PostByMd5(md5: string): Promise { return (await request('/posts', { md5, v2: 'true' })) as E621Post ?? null; } diff --git a/src/utils/event-utils.ts b/src/utils/event-utils.ts index b8726c2..4a20bc4 100644 --- a/src/utils/event-utils.ts +++ b/src/utils/event-utils.ts @@ -1,143 +1,12 @@ import { APIEmbedField, EmbedAuthorOptions } from 'discord.js'; import { config } from '../config'; -import { getE621Post, PostAction, spoilerOrBlacklist } from './e621-utils'; -import { appealIDRegex, blipIDRegex, commentIDRegex, flagIDRegex, forumTopicIDRegex, poolIDRegex, postIDRegex, recordIDRegex, searchLinkRegex, setIDRegex, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from './message-matcher-regex'; -import { parseDTextToMarkdown } from '@clynamic/dmark'; +import { parseDTextToMarkdown } from './dtext-utils'; -// TODO: Condense this and the message event handler regex array. -const linkReplacers = [ - { - regex: blipIDRegex, - replacement: '/blips/{match}', - encodeURI: false - }, - { - regex: commentIDRegex, - replacement: '/comments/{match}', - encodeURI: false - }, - { - regex: forumTopicIDRegex, - replacement: '/forum_topics/{match}', - encodeURI: false - }, - { - regex: poolIDRegex, - replacement: '/pools/{match}', - encodeURI: false - }, - { - regex: postIDRegex, - tester: async (postId: string, before: string, after: string) => { - const post = await getE621Post(postId); - if (!post) return { allowed: true, before, after }; - const allowed = spoilerOrBlacklist(post).action != PostAction.Blacklist; +const MAX_DESCRIPTION_LENGTH = 1024; - return { allowed, before, after }; - }, - replacement: '/posts/{match}', - encodeURI: false - }, - { - regex: recordIDRegex, - replacement: '/user_feedbacks/{match}', - encodeURI: false - }, - { - regex: searchLinkRegex, - replacement: '/posts?tags={match}', - encodeURI: true - }, - { - regex: setIDRegex, - replacement: '/post_sets/{match}', - encodeURI: false - }, - { - regex: takedownIDRegex, - replacement: '/takedowns/{match}', - encodeURI: false - }, - { - regex: ticketIDRegex, - replacement: '/tickets/{match}', - encodeURI: false - }, - { - regex: appealIDRegex, - replacement: '/appeals/{match}', - encodeURI: false - }, - { - regex: flagIDRegex, - replacement: '/flags/{match}', - encodeURI: false - }, - { - regex: userIDRegex, - replacement: '/users/{match}', - encodeURI: false - }, - { - regex: wikiLinkRegex, - replacement: '/wiki_pages/{match}', - encodeURI: true - } -]; - -const urlRegex = new RegExp('"((?:[\\S]| )+?)":\\[?((?:https?:\\/\\/[\\w\\d.\\/?=#&%]+)|\\/[\\w\\d.\\/?=#\\[\\]]+)\\]?', 'gi'); - -const MAX_DESCRIPTION_LENGTH = 500; - -export async function getLinks(input: string, limit: number = Number.MAX_SAFE_INTEGER): Promise { - const length = input.length; - - const replacedIndexes: { start: number, end: number }[] = []; - const checks: Promise<{ allowed: boolean, before: string, after: string }>[] = []; - - for (const replacer of linkReplacers) { - input = input.replaceAll(replacer.regex, (match, group1) => { - const replaced = `[${match}](${config.E621_BASE_URL}${(replacer.replacement).replace('{match}', replacer.encodeURI ? encodeURIComponent(group1) : group1)})`; - if (replacer.tester) checks.push(replacer.tester(group1, match, replaced)); - const start = input.indexOf(match); - replacedIndexes.push({ start, end: start + replaced.length }); - - return replaced; - }); - } - - input = input.replaceAll(urlRegex, (match, group1, group2) => { - const replaced = group2.startsWith('/') ? `[${group1}](${config.E621_BASE_URL}${group2})` : `[${group1}](${group2})`; - const start = input.indexOf(match); - replacedIndexes.push({ start, end: start + replaced.length }); - return replaced; - }); - - const values = await Promise.all(checks); - - for (const check of values) { - if (!check.allowed) { - input = input.replace(check.after, check.before); - } - } - - if (length > limit) { - for (const replacedIndex of replacedIndexes) { - if (replacedIndex.start < limit && replacedIndex.end >= limit) { - return input.substring(0, replacedIndex.end) + '...'; - } - } - - return input.substring(0, limit) + '...'; - } - - return input; -} - -export async function getDescription(data: { reason: string }): Promise { - return parseDTextToMarkdown(data.reason.length <= MAX_DESCRIPTION_LENGTH ? await getLinks(data.reason) : await getLinks(data.reason, MAX_DESCRIPTION_LENGTH), { - baseUrl: 'https://e621.net' - }).output; +export async function parseMarkdownToField(input: string): Promise { + const text = await parseDTextToMarkdown(input); + return text.length >= MAX_DESCRIPTION_LENGTH ? text.slice(0, MAX_DESCRIPTION_LENGTH - 3) + '...' : text; } export function getAuthor(data: { user_id: number, user: string }): EmbedAuthorOptions { diff --git a/src/utils/index.ts b/src/utils/index.ts index af8b12f..00332f9 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -8,6 +8,7 @@ export * from './channel-utils'; export * from './commands'; export * from './debug-utils'; export * from './discord-user-utils'; +export * from './dtext-utils'; export * from './e621-utils'; export * from './embed-utils'; export * from './event-log-utils'; diff --git a/src/utils/ticket-events.ts b/src/utils/ticket-events.ts index 90af9cb..b0063a3 100644 --- a/src/utils/ticket-events.ts +++ b/src/utils/ticket-events.ts @@ -3,7 +3,7 @@ import { config } from '../config'; import { Database } from '../shared/Database'; import { Ticket, TicketPhrase, TicketUpdate } from '../types'; import { PostAction, getE621Post, getE621User, spoilerOrBlacklist } from './e621-utils'; -import { getAuthor, getColor, getDescription, getFields } from './event-utils'; +import { getAuthor, getColor, parseMarkdownToField, getFields } from './event-utils'; import { humanizeCapitalization } from './string-utils'; import { shouldAlert } from './ticket-utils'; @@ -105,10 +105,16 @@ async function createEmbedFromTicket(ticket: Ticket): Promise { return new EmbedBuilder() .setTitle(getTitle(ticket)) .setURL(await getURL(ticket)) - .setDescription(await getDescription(ticket)) .setAuthor(getAuthor(ticket)) .setColor(getColor(ticket)) - .setFields(...getFields(ticket)) + .setFields( + { + name: 'Reason', + value: await parseMarkdownToField(ticket.reason), + inline: false + }, + ...getFields(ticket) + ) .setFooter({ text: `Ticket #${ticket.id}` }); }