Fix markdown parsing and blacklisted appeals

package-lock.json

@@ -5,7 +5,7 @@
   "packages": {
     "": {
       "dependencies": {
-        "@clynamic/dmark": "0.0.8",
+        "@clynamic/dmark": "0.0.11",
         "@redis/client": "5.1.0",
         "body-parser": "2.2.2",
         "discord.js": "14.26.4",
@@ -31,9 +31,9 @@
       }
     },
     "node_modules/@clynamic/dmark": {
-      "version": "0.0.8",
+      "version": "0.0.11",
-      "resolved": "https://npm.clynamic.net/%40clynamic%2Fdmark/-/0.0.8/dmark-0.0.8.tgz",
+      "resolved": "https://npm.clynamic.net/%40clynamic%2Fdmark/-/0.0.11/dmark-0.0.11.tgz",
-      "integrity": "sha512-KzZqTeWldrB2ij74s5hVUVdHaBur4QqYExv6V9mSiTovdcspbRS0pA5v6OwONdxvcBou7b4p4cIalNj4eARgxg==",
+      "integrity": "sha512-efrxWJ6ONQ6kcCLwPfoIonrMznvYYm9fZ1v+DgDY5BzQ8EDrgbCJr3ri9dVS7VUt9Rh8yZTXtaMDI1jf3Gb0Aw==",
       "license": "UNLICENSED"
     },
     "node_modules/@discordjs/builders": {

package.json

@@ -12,7 +12,7 @@
     "typescript-eslint": "8.32.1"
   },
   "dependencies": {
-    "@clynamic/dmark": "0.0.8",
+    "@clynamic/dmark": "0.0.11",
     "@redis/client": "5.1.0",
     "body-parser": "2.2.2",
     "discord.js": "14.26.4",

src/events/handle-message.ts

@@ -50,7 +50,6 @@ const regexTesters = [
 const uniqueRegexMatches = (g, i, a) => a.findIndex(v => v[1] == g[1]) == i;
 
 export async function handleMessageCreate(message: Message) {
-  console.log('message created');
   if (message.author.bot) return;
   if (message.inGuild()) await Database.putMessage(message);
 

src/utils/appeal-events.ts

@@ -2,10 +2,10 @@ import { ActionRowBuilder, APIEmbedField, ButtonBuilder, ButtonStyle, Client, Di
 import { config } from '../config';
 import { Database } from '../shared/Database';
 import { Appeal, AppealUpdate, E621Post, PostFlag } from '../types';
-import { getAuthor, getColor, getDescription, getFields, getLinks } from './event-utils';
+import { getAuthor, getColor, parseMarkdownToField, getFields } from './event-utils';
 import { humanizeCapitalization } from './string-utils';
-import { getE621Post, getE621PostFlag } from './e621-utils';
+import { getE621Post, getE621PostFlag, PostAction, spoilerOrBlacklist } from './e621-utils';
-import { parseDTextToMarkdown } from '@clynamic/dmark';
+import { parseDTextToMarkdown } from './dtext-utils';
 
 export async function appealUpdateHandler(client: Client, update: string) {
   const data: AppealUpdate = JSON.parse(update);
@@ -92,7 +92,7 @@ async function getCustomFields(appeal: Appeal, flag: PostFlag, post: E621Post):
   return [
     {
       name: 'Deletion Reason',
-      value: parseDTextToMarkdown(await getLinks(flag.reason)).output,
+      value: await parseDTextToMarkdown(flag.reason),
       inline: true
     }
   ];
@@ -102,22 +102,31 @@ async function createEmbedFromAppeal(appeal: Appeal, flag: PostFlag, post: E621P
   return new EmbedBuilder()
     .setTitle(getTitle(appeal))
     .setURL(await getURL(appeal))
-    .setDescription(await getDescription(appeal))
     .setAuthor(getAuthor(appeal))
     .setColor(getColor(appeal))
-    .setFields(...getFields(appeal), ...(await getCustomFields(appeal, flag, post)))
+    .setFields(
+      {
+        name: 'Reason',
+        value: await parseMarkdownToField(appeal.reason),
+        inline: false
+      },
+      ...getFields(appeal),
+      ...(await getCustomFields(appeal, flag, post))
+    )
     .setFooter({ text: `Appeal #${appeal.id}` });
 }
 
 async function getButtons(appeal: Appeal, flag: PostFlag, post: E621Post): Promise<ActionRowBuilder<ButtonBuilder>> {
   const row = new ActionRowBuilder<ButtonBuilder>();
 
+  if (spoilerOrBlacklist(post).action != PostAction.Blacklist) {
     const button = new ButtonBuilder()
       .setLabel('Open Post')
       .setStyle(ButtonStyle.Link)
       .setURL(`${config.E621_BASE_URL}/posts/${post.id}`);
 
     row.addComponents(button);
+  }
 
   return row;
 }

src/utils/dtext-utils.ts

@@ -0,0 +1,96 @@
+import { BlockNode, DocumentNode, formatMarkdown, InlineNode, LinkNode, MarkdownFormatContext, MarkdownHandlers, markdownHandlers, parseDTextToAST, TableBodyNode, TableHeadNode, TableLiteralNode, TableRowNode, TextNode } from '@clynamic/dmark';
+import { config } from '../config';
+import { E621Post } from '../types';
+import { getManyE621Posts, PostAction, SEARCH_LIMIT, spoilerOrBlacklist } from './e621-utils';
+
+export async function parseDTextToMarkdown(text: string): Promise<string> {
+  const ast = parseDTextToAST(text, {
+    baseUrl: config.E621_BASE_URL,
+    allowColor: false
+  }) as DocumentNode;
+
+  const postIds: string[][] = [];
+
+  const recurseChildren = (node: BlockNode | InlineNode | TableHeadNode | TableBodyNode | TableRowNode | TableLiteralNode | LinkNode) => {
+
+    if (node.type == 'link' && node.linkType == 'id_link' && node.id) {
+      if (postIds.length == 0 || postIds.at(-1)!.length >= SEARCH_LIMIT) postIds.push([]);
+      postIds.at(-1)!.push(node.id);
+    }
+
+    if ('children' in node && node.children !== undefined) {
+      for (const child of node.children) recurseChildren(child);
+    }
+  };
+
+  for (const child of ast.children) recurseChildren(child);
+
+  const postData: E621Post[] = [];
+
+  for (const chunk of postIds) {
+    postData.push(...await getManyE621Posts(chunk));
+  }
+
+  const handlers: MarkdownHandlers = {
+    ...markdownHandlers,
+
+    link: (node: LinkNode, out: string[], ctx: MarkdownFormatContext) => {
+      ctx.atLineStart = false;
+      if (node.linkType == 'wiki') {
+        formatWikiLink(node, out);
+      } else if (node.linkType == 'id_link' && node.idType == 'post') {
+        const post = postData.find(p => p.id == Number(node.id));
+        if (post && spoilerOrBlacklist(post).action == PostAction.Blacklist) {
+          out.push('post #', node.id!);
+        } else {
+          out.push('[');
+          out.push('post #', node.id!);
+          out.push(']');
+          out.push(`(${node.href.startsWith('/') ? config.E621_BASE_URL : ''}${node.href})`);
+        }
+      } else {
+        out.push('[');
+        markdownHandlers.link(node, out, ctx);
+        out.push(']');
+        out.push(`(${node.href.startsWith('/') ? config.E621_BASE_URL : ''}${node.href})`);
+      }
+    }
+  };
+
+  return formatMarkdown(ast, {}, handlers).output;
+}
+
+
+// Modified from
+// https://github.com/clragon/dmark/blob/main/src/ast/text.ts#L17-L21
+// https://github.com/clragon/dmark/blob/main/src/md/render/index.ts#L481-L531
+const RE_HAS_UPPER = /[A-Z]/;
+export function asciiLowercase(s: string): string {
+  if (!RE_HAS_UPPER.test(s)) return s;
+  return s.replace(/[A-Z]/g, c => String.fromCharCode(c.charCodeAt(0) + 32));
+}
+
+function formatWikiLink(node: LinkNode, out: string[]): void {
+  // Same dispatch as the dtext sibling formatter (ADR-0004). Anchor-only
+  // form has two variants (`[[#anchor]]` and `[[#anchor|title]]`); detect
+  // title-override by comparing children content to the default form.
+  if (node.href.startsWith('#') && node.anchor !== undefined) {
+    const childText
+      = node.children?.[0] && node.children[0].type === 'text'
+        ? (node.children[0] as TextNode).content
+        : '';
+    if (childText === `#${node.anchor}` || childText === '') {
+      out.push('[[#', node.anchor, ']]');
+    } else {
+      out.push('[[#', childText, ']]');
+    }
+    return;
+  }
+
+  const childText
+    = node.children?.[0] && node.children[0].type === 'text'
+      ? (node.children[0] as TextNode).content
+      : '';
+
+  out.push('[', '[[', childText, ']]', ']', `(${config.E621_BASE_URL}${node.href})`);
+}

src/utils/e621-utils.ts

@@ -9,6 +9,8 @@ const SPOILERED_NONSAFE_TAGS: string[] = [];
 
 const USER_AGENT = 'E621DiscordBot';
 
+export const SEARCH_LIMIT = 320;
+
 async function request(path: string, query?: { [name: string]: string }): Promise<any> {
   const url = new URL(config.E621_BASE_URL!);
   url.pathname = path + '.json';
@@ -38,6 +40,10 @@ export async function getE621Post(id: string | number): Promise<E621Post | null>
   return (await request(`/posts/${id}`, { v2: 'true' })) as E621Post ?? null;
 }
 
+export async function getManyE621Posts(ids: string[] | number[]): Promise<E621Post[]> {
+  return (await request('/posts', { v2: 'true', tags: `id:${ids.join(',')}` })) as E621Post[] ?? [];
+}
+
 export async function getE621PostByMd5(md5: string): Promise<E621Post | null> {
   return (await request('/posts', { md5, v2: 'true' })) as E621Post ?? null;
 }

src/utils/event-utils.ts

@@ -1,143 +1,12 @@
 import { APIEmbedField, EmbedAuthorOptions } from 'discord.js';
 import { config } from '../config';
-import { getE621Post, PostAction, spoilerOrBlacklist } from './e621-utils';
+import { parseDTextToMarkdown } from './dtext-utils';
-import { appealIDRegex, blipIDRegex, commentIDRegex, flagIDRegex, forumTopicIDRegex, poolIDRegex, postIDRegex, recordIDRegex, searchLinkRegex, setIDRegex, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from './message-matcher-regex';
-import { parseDTextToMarkdown } from '@clynamic/dmark';
 
-// TODO: Condense this and the message event handler regex array.
+const MAX_DESCRIPTION_LENGTH = 1024;
-const linkReplacers = [
-  {
-    regex: blipIDRegex,
-    replacement: '/blips/{match}',
-    encodeURI: false
-  },
-  {
-    regex: commentIDRegex,
-    replacement: '/comments/{match}',
-    encodeURI: false
-  },
-  {
-    regex: forumTopicIDRegex,
-    replacement: '/forum_topics/{match}',
-    encodeURI: false
-  },
-  {
-    regex: poolIDRegex,
-    replacement: '/pools/{match}',
-    encodeURI: false
-  },
-  {
-    regex: postIDRegex,
-    tester: async (postId: string, before: string, after: string) => {
-      const post = await getE621Post(postId);
-      if (!post) return { allowed: true, before, after };
-      const allowed = spoilerOrBlacklist(post).action != PostAction.Blacklist;
 
-      return { allowed, before, after };
+export async function parseMarkdownToField(input: string): Promise<string> {
-    },
+  const text = await parseDTextToMarkdown(input);
-    replacement: '/posts/{match}',
+  return text.length >= MAX_DESCRIPTION_LENGTH ? text.slice(0, MAX_DESCRIPTION_LENGTH - 3) + '...' : text;
-    encodeURI: false
-  },
-  {
-    regex: recordIDRegex,
-    replacement: '/user_feedbacks/{match}',
-    encodeURI: false
-  },
-  {
-    regex: searchLinkRegex,
-    replacement: '/posts?tags={match}',
-    encodeURI: true
-  },
-  {
-    regex: setIDRegex,
-    replacement: '/post_sets/{match}',
-    encodeURI: false
-  },
-  {
-    regex: takedownIDRegex,
-    replacement: '/takedowns/{match}',
-    encodeURI: false
-  },
-  {
-    regex: ticketIDRegex,
-    replacement: '/tickets/{match}',
-    encodeURI: false
-  },
-  {
-    regex: appealIDRegex,
-    replacement: '/appeals/{match}',
-    encodeURI: false
-  },
-  {
-    regex: flagIDRegex,
-    replacement: '/flags/{match}',
-    encodeURI: false
-  },
-  {
-    regex: userIDRegex,
-    replacement: '/users/{match}',
-    encodeURI: false
-  },
-  {
-    regex: wikiLinkRegex,
-    replacement: '/wiki_pages/{match}',
-    encodeURI: true
-  }
-];
-
-const urlRegex = new RegExp('"((?:[\\S]| )+?)":\\[?((?:https?:\\/\\/[\\w\\d.\\/?=#&%]+)|\\/[\\w\\d.\\/?=#\\[\\]]+)\\]?', 'gi');
-
-const MAX_DESCRIPTION_LENGTH = 500;
-
-export async function getLinks(input: string, limit: number = Number.MAX_SAFE_INTEGER): Promise<string> {
-  const length = input.length;
-
-  const replacedIndexes: { start: number, end: number }[] = [];
-  const checks: Promise<{ allowed: boolean, before: string, after: string }>[] = [];
-
-  for (const replacer of linkReplacers) {
-    input = input.replaceAll(replacer.regex, (match, group1) => {
-      const replaced = `[${match}](${config.E621_BASE_URL}${(replacer.replacement).replace('{match}', replacer.encodeURI ? encodeURIComponent(group1) : group1)})`;
-      if (replacer.tester) checks.push(replacer.tester(group1, match, replaced));
-      const start = input.indexOf(match);
-      replacedIndexes.push({ start, end: start + replaced.length });
-
-      return replaced;
-    });
-  }
-
-  input = input.replaceAll(urlRegex, (match, group1, group2) => {
-    const replaced = group2.startsWith('/') ? `[${group1}](${config.E621_BASE_URL}${group2})` : `[${group1}](${group2})`;
-    const start = input.indexOf(match);
-    replacedIndexes.push({ start, end: start + replaced.length });
-    return replaced;
-  });
-
-  const values = await Promise.all(checks);
-
-  for (const check of values) {
-    if (!check.allowed) {
-      input = input.replace(check.after, check.before);
-    }
-  }
-
-  if (length > limit) {
-    for (const replacedIndex of replacedIndexes) {
-      if (replacedIndex.start < limit && replacedIndex.end >= limit) {
-        return input.substring(0, replacedIndex.end) + '...';
-      }
-    }
-
-    return input.substring(0, limit) + '...';
-  }
-
-  return input;
-}
-
-export async function getDescription(data: { reason: string }): Promise<string> {
-  return parseDTextToMarkdown(data.reason.length <= MAX_DESCRIPTION_LENGTH ? await getLinks(data.reason) : await getLinks(data.reason, MAX_DESCRIPTION_LENGTH), {
-    baseUrl: 'https://e621.net'
-  }).output;
 }
 
 export function getAuthor(data: { user_id: number, user: string }): EmbedAuthorOptions {

src/utils/index.ts

@@ -8,6 +8,7 @@ export * from './channel-utils';
 export * from './commands';
 export * from './debug-utils';
 export * from './discord-user-utils';
+export * from './dtext-utils';
 export * from './e621-utils';
 export * from './embed-utils';
 export * from './event-log-utils';

src/utils/ticket-events.ts

@@ -3,7 +3,7 @@ import { config } from '../config';
 import { Database } from '../shared/Database';
 import { Ticket, TicketPhrase, TicketUpdate } from '../types';
 import { PostAction, getE621Post, getE621User, spoilerOrBlacklist } from './e621-utils';
-import { getAuthor, getColor, getDescription, getFields } from './event-utils';
+import { getAuthor, getColor, parseMarkdownToField, getFields } from './event-utils';
 import { humanizeCapitalization } from './string-utils';
 import { shouldAlert } from './ticket-utils';
 
@@ -105,10 +105,16 @@ async function createEmbedFromTicket(ticket: Ticket): Promise<EmbedBuilder> {
   return new EmbedBuilder()
     .setTitle(getTitle(ticket))
     .setURL(await getURL(ticket))
-    .setDescription(await getDescription(ticket))
     .setAuthor(getAuthor(ticket))
     .setColor(getColor(ticket))
-    .setFields(...getFields(ticket))
+    .setFields(
+      {
+        name: 'Reason',
+        value: await parseMarkdownToField(ticket.reason),
+        inline: false
+      },
+      ...getFields(ticket)
+    )
     .setFooter({ text: `Ticket #${ticket.id}` });
 }