Revert "Remove database-related message content usage"

This reverts commit 0b211db023.

sql/migrations/0002-remove-messages.sql

@@ -1,22 +0,0 @@
---------------------------------------------------------------------------------
--- Up
---------------------------------------------------------------------------------
-DROP TABLE messages;
-
---------------------------------------------------------------------------------
--- Down
---------------------------------------------------------------------------------
-CREATE TABLE
-  IF NOT EXISTS messages (
-    id TEXT PRIMARY KEY ON CONFLICT REPLACE,
-    author_id TEXT NOT NULL,
-    author_name TEXT NOT NULL,
-    channel_id TEXT NOT NULL,
-    attachments TEXT NOT NULL,
-    stickers TEXT NOT NULL,
-    content TEXT NOT NULL
-  );
-
-CREATE INDEX IF NOT EXISTS index_authors ON messages (author_id);
-
-CREATE INDEX IF NOT EXISTS index_channels ON messages (channel_id);

sql/structure.sql

@@ -28,6 +28,21 @@ CREATE TABLE
     private_help_channel_id TEXT
   );
 
+CREATE TABLE
+  IF NOT EXISTS messages (
+    id TEXT PRIMARY KEY ON CONFLICT REPLACE,
+    author_id TEXT NOT NULL,
+    author_name TEXT NOT NULL,
+    channel_id TEXT NOT NULL,
+    attachments TEXT NOT NULL,
+    stickers TEXT NOT NULL,
+    content TEXT NOT NULL
+  );
+
+CREATE INDEX IF NOT EXISTS index_authors ON messages (author_id);
+
+CREATE INDEX IF NOT EXISTS index_channels ON messages (channel_id);
+
 CREATE TABLE
   IF NOT EXISTS tickets (id INTEGER PRIMARY KEY, message_id TEXT NOT NULL);
 

src/events/handle-message.ts

@@ -1,7 +1,8 @@
-import { Message as DiscordMessage, GuildBasedChannel, GuildTextBasedChannel, OmitPartialGroupDMChannel, PartialMessage, spoiler } from 'discord.js';
+import { Message as DiscordMessage, GuildBasedChannel, GuildTextBasedChannel, OmitPartialGroupDMChannel, PartialMessage, ReadonlyCollection, spoiler } from 'discord.js';
 import { config } from '../config';
+import { Database } from '../shared/Database';
 import { E621Post } from '../types';
-import { ALLOWED_MIMETYPES, appealIDRegex, artistIDRegex, blipIDRegex, calculateMD5FromURL, channelIgnoresLinks, channelIsInStaffCategory, channelIsSafe, commentIDRegex, flagIDRegex, forumTopicIDRegex, getE621Post, getE621PostByMd5, getPostUrl, isInSpoilerTags, issueRegex, poolIDRegex, PostAction, postIDRegex, prRegex, recordIDRegex, searchLinkRegex, setIDRegex, spoilerOrBlacklist, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from '../utils';
+import { ALLOWED_MIMETYPES, appealIDRegex, artistIDRegex, blipIDRegex, calculateMD5FromURL, channelIgnoresLinks, channelIsInStaffCategory, channelIsSafe, commentIDRegex, flagIDRegex, forumTopicIDRegex, getE621Post, getE621PostByMd5, getPostUrl, isEdited, isInSpoilerTags, issueRegex, logDeletion, logEdit, poolIDRegex, PostAction, postIDRegex, prRegex, recordIDRegex, searchLinkRegex, setIDRegex, spoilerOrBlacklist, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from '../utils';
 
 export type Message<InGuild extends boolean = boolean> = OmitPartialGroupDMChannel<DiscordMessage<InGuild>>;
 export type Partial = OmitPartialGroupDMChannel<PartialMessage>;
@@ -49,7 +50,9 @@ const regexTesters = [
 const uniqueRegexMatches = (g, i, a) => a.findIndex(v => v[1] == g[1]) == i;
 
 export async function handleMessageCreate(message: Message) {
+  console.log('message created');
   if (message.author.bot) return;
+  if (message.inGuild()) await Database.putMessage(message);
 
   const responses: string[] = [];
 
@@ -110,9 +113,22 @@ export async function handleMessageCreate(message: Message) {
 }
 
 export async function handleMessageUpdate(oldMessage: Message | PartialMessage, newMessage: Message) {
-  if (newMessage.author.bot || !oldMessage.content) return;
+  if (newMessage.author.bot) return;
 
-  if (oldMessage.content == newMessage.content) return;
+  const loggedMessage = await Database.getMessageWithRetry(newMessage.id);
+
+  if (!loggedMessage) {
+    if (newMessage.inGuild()) await Database.putMessage(newMessage);
+
+    return;
+  }
+
+  if (newMessage.inGuild() && isEdited(loggedMessage, newMessage)) {
+    await Database.putMessage(newMessage);
+    await logEdit(loggedMessage, newMessage);
+  }
+
+  if (loggedMessage.content == newMessage.content) return;
 
   const responses: string[] = [];
 
@@ -132,7 +148,7 @@ export async function handleMessageUpdate(oldMessage: Message | PartialMessage,
       }
       test.regex.lastIndex = 0;
 
-      while ((match = test.regex.exec(oldMessage.content)) != null) {
+      while ((match = test.regex.exec(loggedMessage.content)) != null) {
         oldMatches.push(match);
       }
       test.regex.lastIndex = 0;
@@ -158,6 +174,20 @@ export async function handleMessageUpdate(oldMessage: Message | PartialMessage,
   }
 }
 
+export async function handleMessageDelete(message: Message | PartialMessage) {
+  const loggedMessage = await Database.getMessageWithRetry(message.id);
+
+  if (!loggedMessage) return;
+
+  if (message.inGuild()) await logDeletion(loggedMessage, message);
+}
+
+export async function handleBulkMessageDelete(messages: ReadonlyCollection<string, Message | Partial>, channel: GuildTextBasedChannel) {
+  for (const message of messages.values()) {
+    await handleMessageDelete(message);
+  }
+}
+
 async function searchHandler(message: Message, matchedGroups: RegExpExecArray[]): Promise<string | boolean> {
   const skip = await channelIgnoresLinks(message.channel as GuildBasedChannel);
 

src/index.ts

@@ -1,6 +1,6 @@
 import { Client as DiscordClient, GatewayIntentBits, MessageFlags, Partials } from 'discord.js';
 import { config } from './config';
-import { handleAuditLogCreate, handleBanRemove, handleGuildCreate, handleMemberJoin, handleMessageCreate, handleMessageUpdate, handleThreadCreate, handleVoiceStateUpdate } from './events';
+import { handleAuditLogCreate, handleBanRemove, handleBulkMessageDelete, handleGuildCreate, handleMemberJoin, handleMessageCreate, handleMessageDelete, handleMessageUpdate, handleThreadCreate, handleVoiceStateUpdate } from './events';
 import { ScheduledTasks, Scheduler } from './scheduler';
 import { Database } from './shared/Database';
 import { openRedisClient } from './shared/RedisClient';
@@ -145,6 +145,8 @@ client.on('clientReady', async () => {
   client.on('guildCreate', handleGuildCreate);
   client.on('guildMemberAdd', handleMemberJoin);
   client.on('messageCreate', handleMessageCreate);
+  client.on('messageDelete', handleMessageDelete);
+  client.on('messageDeleteBulk', handleBulkMessageDelete);
   client.on('messageUpdate', handleMessageUpdate);
   client.on('threadCreate', handleThreadCreate);
   client.on('voiceStateUpdate', handleVoiceStateUpdate);

src/modals/report-message.ts

@@ -1,4 +1,4 @@
-import { ActionRowBuilder, ButtonBuilder, ButtonStyle, Client, EmbedBuilder, GuildTextBasedChannel, MessageFlags, ModalSubmitInteraction } from 'discord.js';
+import { ActionRowBuilder, AttachmentBuilder, ButtonBuilder, ButtonStyle, Client, EmbedBuilder, GuildTextBasedChannel, MessageFlags, ModalSubmitInteraction } from 'discord.js';
 import { Database } from '../shared/Database';
 import { canOpenPrivateHelpTicket, createPrivateHelpTicketThread } from '../utils';
 
@@ -86,6 +86,8 @@ export default {
       components: createPrivateHelpTicket && !wantsTicketButCantOpen ? [] : [row]
     });
 
+    await reportsChannel.send({ files: [new AttachmentBuilder(Buffer.from(reportedMessage.content), { name: 'message-content.txt' })] });
+
     if (createPrivateHelpTicket && !wantsTicketButCantOpen) {
       if (!guildSettings.private_help_channel_id) {
         replyContent += ' Could not open private help ticket. Private help channel not set. Please report this to a staff member.';

src/shared/Database.ts

@@ -1,9 +1,11 @@
-import { readFileSync } from 'fs';
 import path, { dirname } from 'path';
 import { open, Database as SqliteDatabase } from 'sqlite';
 import sqlite3 from 'sqlite3';
 import { fileURLToPath } from 'url';
-import { AppealMessage, Ban, GithubUserMapping, GuildArraySetting, GuildSetting, GuildSettings, KnowledgebaseItem, Note, PrivateHelpTicket, TicketMessage, TicketPhrase } from '../types';
+import { Message } from '../events';
+import { AppealMessage, Ban, GithubUserMapping, GuildArraySetting, GuildSetting, GuildSettings, KnowledgebaseItem, LoggedMessage, Note, PrivateHelpTicket, TicketMessage, TicketPhrase } from '../types';
+import { serializeMessage, wait } from '../utils';
+import { readFileSync } from 'fs';
 
 export const enum PrivateHelpTicketStatus {
   OPEN = 0,
@@ -138,6 +140,42 @@ export class Database {
 
   //#endregion
 
+  //#region Message Logs
+
+  static async putMessage(message: Message): Promise<boolean> {
+    try {
+      const serializedMessage = serializeMessage(message);
+
+      await Database.db.run(`
+        INSERT INTO messages (id, author_id, author_name, channel_id, attachments, stickers, content) VALUES
+        (:id, :author_id, :author_name, :channel_id, :attachments, :stickers, :content)
+      `, ...serializedMessage);
+
+      return true;
+    } catch (e) {
+      console.error(e);
+      return false;
+    }
+  }
+
+  static async getMessage(id: string): Promise<LoggedMessage | undefined> {
+    return await Database.db.get<LoggedMessage>('SELECT * FROM messages WHERE id = ?', id);
+  }
+
+  static async getMessageWithRetry(id: string, retries = 5, delay = 500): Promise<LoggedMessage | undefined> {
+    let tried = 0;
+    while (tried < retries) {
+      tried++;
+      const message = await Database.db.get<LoggedMessage>('SELECT * FROM messages WHERE id = ?', id);
+
+      if (message) return message;
+
+      await wait(delay);
+    }
+  }
+
+  //#endregion
+
   //#region Tickets
 
   static async putTicket(ticketId: number, messageId: string) {

src/types/database-types.d.ts

@@ -1,3 +1,13 @@
+export type LoggedMessage = {
+  id: string
+  author_id: string
+  author_name: string
+  channel_id: string
+  attachments: string
+  stickers: string
+  content: string
+}
+
 export type GuildSettings = {
   guild_id: string
   general_chat_id?: string

src/utils/event-log-utils.ts

@@ -1,6 +1,9 @@
-import { APIEmbedField, EmbedBuilder, Guild, GuildBasedChannel, GuildTextBasedChannel } from 'discord.js';
+import { APIEmbedField, AttachmentBuilder, EmbedBuilder, Guild, GuildBasedChannel, GuildTextBasedChannel, MessageCreateOptions } from 'discord.js';
+import { Message } from '../events';
 import { Database } from '../shared/Database';
+import { LoggedMessage } from '../types';
 import { channelIsInStaffCategory } from './channel-utils';
+import { deserializeMessagePart, getModifiedAttachments, getModifiedStickers } from './message-utils';
 
 type CustomEventLogData = {
   title: string
@@ -10,6 +13,63 @@ type CustomEventLogData = {
   fields: APIEmbedField[] | null
 }
 
+export async function logEdit(loggedMessage: LoggedMessage, newMessage: Message<true>) {
+  const channel = await getEventLogChannel(newMessage.guild, newMessage.channel);
+
+  if (!channel) return;
+
+  const includeContentInEmbed = loggedMessage.content.length <= 1024 && newMessage.content.length <= 1024;
+
+  const fields: APIEmbedField[] = [];
+  fields.push(...getMainEmbeds(loggedMessage, newMessage));
+  fields.push(...getEditEmbeds(loggedMessage, newMessage, includeContentInEmbed));
+
+  const embed = new EmbedBuilder()
+    .setTitle('Edited Message')
+    .setColor(0xFFFF00)
+    .setTimestamp(newMessage.createdTimestamp)
+    .addFields(...fields);
+
+  const messagePayload: MessageCreateOptions = { embeds: [embed] };
+
+  if (!includeContentInEmbed) {
+    const before = new AttachmentBuilder(Buffer.from(loggedMessage.content), { name: 'before.txt' });
+    const after = new AttachmentBuilder(Buffer.from(newMessage.content), { name: 'after.txt' });
+
+    messagePayload.files = [before, after];
+  }
+
+  channel.send(messagePayload);
+}
+
+export async function logDeletion(loggedMessage: LoggedMessage, deletedMessage: Message<true>) {
+  const channel = await getEventLogChannel(deletedMessage.guild, deletedMessage.channel);
+
+  if (!channel) return;
+
+  const includeContentInEmbed = loggedMessage.content.length <= 1024;
+
+  const fields: APIEmbedField[] = [];
+  fields.push(...getMainEmbeds(loggedMessage, deletedMessage));
+  fields.push(...getDeletedEmbeds(loggedMessage, includeContentInEmbed));
+
+  const embed = new EmbedBuilder()
+    .setTitle('Deleted Message')
+    .setColor(0xFF0000)
+    .setTimestamp(deletedMessage.createdTimestamp)
+    .addFields(...fields);
+
+  const messagePayload: MessageCreateOptions = { embeds: [embed] };
+
+  if (!includeContentInEmbed) {
+    const before = new AttachmentBuilder(Buffer.from(loggedMessage.content), { name: 'content.txt' });
+
+    messagePayload.files = [before];
+  }
+
+  channel.send(messagePayload);
+}
+
 export async function logCustomEvent(guild: Guild, data: CustomEventLogData) {
   const channel = await getEventLogChannel(guild);
 
@@ -47,4 +107,113 @@ async function getEventLogChannel(guild: Guild, channel: GuildBasedChannel | nul
 
     return channel;
   }
+}
+
+function getMainEmbeds(loggedMessage: LoggedMessage, newMessage: Message<true>): APIEmbedField[] {
+  const channelString = `${newMessage.channel.toString()}\n${newMessage.channel.name}`;
+  const userString = `<@${loggedMessage.author_id}>\n${loggedMessage.author_name}`;
+
+  return [
+    {
+      name: 'Channel',
+      value: channelString,
+      inline: true
+    },
+    {
+      name: 'User',
+      value: userString,
+      inline: true
+    },
+    {
+      name: 'Message',
+      value: `[${newMessage.id}](${newMessage.url})`,
+      inline: true
+    },
+  ];
+}
+
+function getDeletedEmbeds(loggedMessage: LoggedMessage, includeContentInEmbed = true): APIEmbedField[] {
+  const fields: APIEmbedField[] = [];
+
+  if (includeContentInEmbed && loggedMessage.content != '') {
+    fields.push({
+      name: 'Content',
+      value: loggedMessage.content,
+      inline: false
+    });
+  }
+
+  for (const attachment of deserializeMessagePart(loggedMessage.attachments)) {
+    fields.push({
+      name: 'Attachment',
+      value: attachment,
+      inline: true
+    });
+  }
+
+  for (const sticker of deserializeMessagePart(loggedMessage.stickers)) {
+    fields.push({
+      name: 'Stickers',
+      value: sticker,
+      inline: true
+    });
+  }
+
+  return fields;
+}
+
+function getEditEmbeds(loggedMessage: LoggedMessage, newMessage: Message<true>, includeContentInEmbed = true): APIEmbedField[] {
+  const fields: APIEmbedField[] = [];
+  if (includeContentInEmbed && loggedMessage.content != newMessage.content) {
+    fields.push(
+      {
+        name: 'Before',
+        value: loggedMessage.content,
+        inline: false
+      },
+      {
+        name: 'After',
+        value: newMessage.content,
+        inline: false
+      }
+    );
+  }
+
+  const { addedAttachments, removedAttachments } = getModifiedAttachments(loggedMessage, newMessage);
+
+  for (const removedAttachment of removedAttachments) {
+    fields.push({
+      name: 'Removed Attachment',
+      value: removedAttachment,
+      inline: true
+    });
+  }
+
+  for (const addedAttachment of addedAttachments) {
+    fields.push({
+      name: 'Added Attachment',
+      value: addedAttachment,
+      inline: true
+    });
+  }
+
+  const { addedStickers, removedStickers } = getModifiedStickers(loggedMessage, newMessage);
+
+  for (const removedSticker of addedStickers) {
+    fields.push({
+      name: 'Removed Sticker',
+      value: removedSticker,
+      inline: true
+    });
+  }
+
+  for (const addedSticker of removedStickers) {
+    fields.push({
+      name: 'Added Sticker',
+      value: addedSticker,
+      inline: true
+    });
+  }
+
+  return fields;
 }

src/utils/message-utils.ts

@@ -1,4 +1,5 @@
 import { Message } from '../events';
+import { LoggedMessage } from '../types';
 
 export const ARRAY_SEPARATOR = '$';
 
@@ -15,6 +16,34 @@ export function deserializeMessagePart(part: string): string[] {
   return part.split(ARRAY_SEPARATOR).filter(e => e);
 }
 
+export function getModifiedAttachments(loggedMessage: LoggedMessage, newMessage: Message): { addedAttachments: string[], removedAttachments: string[] } {
+  const loggedAttachments = loggedMessage.attachments.split(ARRAY_SEPARATOR).filter(e => e);
+  const addedAttachments = newMessage.attachments.filter(a => !loggedAttachments.includes(`${a.name}:${a.id}`)).map(a => `${a.name}:${a.id}`);
+  const removedAttachments = loggedAttachments.filter(a => !newMessage.attachments.has(a.split(':').at(-1)!));
+
+  return { addedAttachments, removedAttachments };
+}
+
+export function getModifiedStickers(loggedMessage: LoggedMessage, newMessage: Message): { addedStickers: string[], removedStickers: string[] } {
+  const loggedStickers = loggedMessage.stickers.split(ARRAY_SEPARATOR).filter(e => e);
+  const addedStickers = newMessage.stickers.filter(s => !loggedStickers.includes(`${s.name}:${s.id}`)).map(s => `${s.name}:${s.id}`);
+  const removedStickers = loggedStickers.filter(s => !newMessage.stickers.has(s.split(':').at(-1)!));
+
+  return { addedStickers, removedStickers };
+}
+
+export function isEdited(loggedMessage: LoggedMessage, newMessage: Message) {
+  if (newMessage.content != loggedMessage.content) return true;
+
+  const { addedAttachments, removedAttachments } = getModifiedAttachments(loggedMessage, newMessage);
+
+  if (addedAttachments.length > 0 || removedAttachments.length > 0) return true;
+
+  const { addedStickers, removedStickers } = getModifiedStickers(loggedMessage, newMessage);
+
+  return addedStickers.length > 0 || removedStickers.length > 0;
+}
+
 export function isInSpoilerTags(content: string, index: number): boolean {
   if (!content.includes('||')) return false;