diff --git a/sql/migrations/0002-remove-messages.sql b/sql/migrations/0002-remove-messages.sql deleted file mode 100644 index 152e929..0000000 --- a/sql/migrations/0002-remove-messages.sql +++ /dev/null @@ -1,22 +0,0 @@ --------------------------------------------------------------------------------- --- Up --------------------------------------------------------------------------------- -DROP TABLE messages; - --------------------------------------------------------------------------------- --- Down --------------------------------------------------------------------------------- -CREATE TABLE - IF NOT EXISTS messages ( - id TEXT PRIMARY KEY ON CONFLICT REPLACE, - author_id TEXT NOT NULL, - author_name TEXT NOT NULL, - channel_id TEXT NOT NULL, - attachments TEXT NOT NULL, - stickers TEXT NOT NULL, - content TEXT NOT NULL - ); - -CREATE INDEX IF NOT EXISTS index_authors ON messages (author_id); - -CREATE INDEX IF NOT EXISTS index_channels ON messages (channel_id); \ No newline at end of file diff --git a/sql/structure.sql b/sql/structure.sql index 2c62628..2eae168 100644 --- a/sql/structure.sql +++ b/sql/structure.sql @@ -28,6 +28,21 @@ CREATE TABLE private_help_channel_id TEXT ); +CREATE TABLE + IF NOT EXISTS messages ( + id TEXT PRIMARY KEY ON CONFLICT REPLACE, + author_id TEXT NOT NULL, + author_name TEXT NOT NULL, + channel_id TEXT NOT NULL, + attachments TEXT NOT NULL, + stickers TEXT NOT NULL, + content TEXT NOT NULL + ); + +CREATE INDEX IF NOT EXISTS index_authors ON messages (author_id); + +CREATE INDEX IF NOT EXISTS index_channels ON messages (channel_id); + CREATE TABLE IF NOT EXISTS tickets (id INTEGER PRIMARY KEY, message_id TEXT NOT NULL); diff --git a/src/events/handle-message.ts b/src/events/handle-message.ts index 47ddf92..d5a633f 100644 --- a/src/events/handle-message.ts +++ b/src/events/handle-message.ts @@ -1,7 +1,8 @@ -import { Message as DiscordMessage, GuildBasedChannel, GuildTextBasedChannel, OmitPartialGroupDMChannel, PartialMessage, spoiler } from 'discord.js'; +import { Message as DiscordMessage, GuildBasedChannel, GuildTextBasedChannel, OmitPartialGroupDMChannel, PartialMessage, ReadonlyCollection, spoiler } from 'discord.js'; import { config } from '../config'; +import { Database } from '../shared/Database'; import { E621Post } from '../types'; -import { ALLOWED_MIMETYPES, appealIDRegex, artistIDRegex, blipIDRegex, calculateMD5FromURL, channelIgnoresLinks, channelIsInStaffCategory, channelIsSafe, commentIDRegex, flagIDRegex, forumTopicIDRegex, getE621Post, getE621PostByMd5, getPostUrl, isInSpoilerTags, issueRegex, poolIDRegex, PostAction, postIDRegex, prRegex, recordIDRegex, searchLinkRegex, setIDRegex, spoilerOrBlacklist, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from '../utils'; +import { ALLOWED_MIMETYPES, appealIDRegex, artistIDRegex, blipIDRegex, calculateMD5FromURL, channelIgnoresLinks, channelIsInStaffCategory, channelIsSafe, commentIDRegex, flagIDRegex, forumTopicIDRegex, getE621Post, getE621PostByMd5, getPostUrl, isEdited, isInSpoilerTags, issueRegex, logDeletion, logEdit, poolIDRegex, PostAction, postIDRegex, prRegex, recordIDRegex, searchLinkRegex, setIDRegex, spoilerOrBlacklist, takedownIDRegex, ticketIDRegex, userIDRegex, wikiLinkRegex } from '../utils'; export type Message = OmitPartialGroupDMChannel>; export type Partial = OmitPartialGroupDMChannel; @@ -49,7 +50,9 @@ const regexTesters = [ const uniqueRegexMatches = (g, i, a) => a.findIndex(v => v[1] == g[1]) == i; export async function handleMessageCreate(message: Message) { + console.log('message created'); if (message.author.bot) return; + if (message.inGuild()) await Database.putMessage(message); const responses: string[] = []; @@ -110,9 +113,22 @@ export async function handleMessageCreate(message: Message) { } export async function handleMessageUpdate(oldMessage: Message | PartialMessage, newMessage: Message) { - if (newMessage.author.bot || !oldMessage.content) return; + if (newMessage.author.bot) return; - if (oldMessage.content == newMessage.content) return; + const loggedMessage = await Database.getMessageWithRetry(newMessage.id); + + if (!loggedMessage) { + if (newMessage.inGuild()) await Database.putMessage(newMessage); + + return; + } + + if (newMessage.inGuild() && isEdited(loggedMessage, newMessage)) { + await Database.putMessage(newMessage); + await logEdit(loggedMessage, newMessage); + } + + if (loggedMessage.content == newMessage.content) return; const responses: string[] = []; @@ -132,7 +148,7 @@ export async function handleMessageUpdate(oldMessage: Message | PartialMessage, } test.regex.lastIndex = 0; - while ((match = test.regex.exec(oldMessage.content)) != null) { + while ((match = test.regex.exec(loggedMessage.content)) != null) { oldMatches.push(match); } test.regex.lastIndex = 0; @@ -158,6 +174,20 @@ export async function handleMessageUpdate(oldMessage: Message | PartialMessage, } } +export async function handleMessageDelete(message: Message | PartialMessage) { + const loggedMessage = await Database.getMessageWithRetry(message.id); + + if (!loggedMessage) return; + + if (message.inGuild()) await logDeletion(loggedMessage, message); +} + +export async function handleBulkMessageDelete(messages: ReadonlyCollection, channel: GuildTextBasedChannel) { + for (const message of messages.values()) { + await handleMessageDelete(message); + } +} + async function searchHandler(message: Message, matchedGroups: RegExpExecArray[]): Promise { const skip = await channelIgnoresLinks(message.channel as GuildBasedChannel); diff --git a/src/index.ts b/src/index.ts index a714f92..23209ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import { Client as DiscordClient, GatewayIntentBits, MessageFlags, Partials } from 'discord.js'; import { config } from './config'; -import { handleAuditLogCreate, handleBanRemove, handleGuildCreate, handleMemberJoin, handleMessageCreate, handleMessageUpdate, handleThreadCreate, handleVoiceStateUpdate } from './events'; +import { handleAuditLogCreate, handleBanRemove, handleBulkMessageDelete, handleGuildCreate, handleMemberJoin, handleMessageCreate, handleMessageDelete, handleMessageUpdate, handleThreadCreate, handleVoiceStateUpdate } from './events'; import { ScheduledTasks, Scheduler } from './scheduler'; import { Database } from './shared/Database'; import { openRedisClient } from './shared/RedisClient'; @@ -145,6 +145,8 @@ client.on('clientReady', async () => { client.on('guildCreate', handleGuildCreate); client.on('guildMemberAdd', handleMemberJoin); client.on('messageCreate', handleMessageCreate); + client.on('messageDelete', handleMessageDelete); + client.on('messageDeleteBulk', handleBulkMessageDelete); client.on('messageUpdate', handleMessageUpdate); client.on('threadCreate', handleThreadCreate); client.on('voiceStateUpdate', handleVoiceStateUpdate); diff --git a/src/modals/report-message.ts b/src/modals/report-message.ts index 3bab83e..139b193 100644 --- a/src/modals/report-message.ts +++ b/src/modals/report-message.ts @@ -1,4 +1,4 @@ -import { ActionRowBuilder, ButtonBuilder, ButtonStyle, Client, EmbedBuilder, GuildTextBasedChannel, MessageFlags, ModalSubmitInteraction } from 'discord.js'; +import { ActionRowBuilder, AttachmentBuilder, ButtonBuilder, ButtonStyle, Client, EmbedBuilder, GuildTextBasedChannel, MessageFlags, ModalSubmitInteraction } from 'discord.js'; import { Database } from '../shared/Database'; import { canOpenPrivateHelpTicket, createPrivateHelpTicketThread } from '../utils'; @@ -86,6 +86,8 @@ export default { components: createPrivateHelpTicket && !wantsTicketButCantOpen ? [] : [row] }); + await reportsChannel.send({ files: [new AttachmentBuilder(Buffer.from(reportedMessage.content), { name: 'message-content.txt' })] }); + if (createPrivateHelpTicket && !wantsTicketButCantOpen) { if (!guildSettings.private_help_channel_id) { replyContent += ' Could not open private help ticket. Private help channel not set. Please report this to a staff member.'; diff --git a/src/shared/Database.ts b/src/shared/Database.ts index d474fa7..348e0af 100644 --- a/src/shared/Database.ts +++ b/src/shared/Database.ts @@ -1,9 +1,11 @@ -import { readFileSync } from 'fs'; import path, { dirname } from 'path'; import { open, Database as SqliteDatabase } from 'sqlite'; import sqlite3 from 'sqlite3'; import { fileURLToPath } from 'url'; -import { AppealMessage, Ban, GithubUserMapping, GuildArraySetting, GuildSetting, GuildSettings, KnowledgebaseItem, Note, PrivateHelpTicket, TicketMessage, TicketPhrase } from '../types'; +import { Message } from '../events'; +import { AppealMessage, Ban, GithubUserMapping, GuildArraySetting, GuildSetting, GuildSettings, KnowledgebaseItem, LoggedMessage, Note, PrivateHelpTicket, TicketMessage, TicketPhrase } from '../types'; +import { serializeMessage, wait } from '../utils'; +import { readFileSync } from 'fs'; export const enum PrivateHelpTicketStatus { OPEN = 0, @@ -138,6 +140,42 @@ export class Database { //#endregion + //#region Message Logs + + static async putMessage(message: Message): Promise { + try { + const serializedMessage = serializeMessage(message); + + await Database.db.run(` + INSERT INTO messages (id, author_id, author_name, channel_id, attachments, stickers, content) VALUES + (:id, :author_id, :author_name, :channel_id, :attachments, :stickers, :content) + `, ...serializedMessage); + + return true; + } catch (e) { + console.error(e); + return false; + } + } + + static async getMessage(id: string): Promise { + return await Database.db.get('SELECT * FROM messages WHERE id = ?', id); + } + + static async getMessageWithRetry(id: string, retries = 5, delay = 500): Promise { + let tried = 0; + while (tried < retries) { + tried++; + const message = await Database.db.get('SELECT * FROM messages WHERE id = ?', id); + + if (message) return message; + + await wait(delay); + } + } + + //#endregion + //#region Tickets static async putTicket(ticketId: number, messageId: string) { diff --git a/src/types/database-types.d.ts b/src/types/database-types.d.ts index 694b9e2..a78a53e 100644 --- a/src/types/database-types.d.ts +++ b/src/types/database-types.d.ts @@ -1,3 +1,13 @@ +export type LoggedMessage = { + id: string + author_id: string + author_name: string + channel_id: string + attachments: string + stickers: string + content: string +} + export type GuildSettings = { guild_id: string general_chat_id?: string diff --git a/src/utils/event-log-utils.ts b/src/utils/event-log-utils.ts index 4bb9bd8..b67edfc 100644 --- a/src/utils/event-log-utils.ts +++ b/src/utils/event-log-utils.ts @@ -1,6 +1,9 @@ -import { APIEmbedField, EmbedBuilder, Guild, GuildBasedChannel, GuildTextBasedChannel } from 'discord.js'; +import { APIEmbedField, AttachmentBuilder, EmbedBuilder, Guild, GuildBasedChannel, GuildTextBasedChannel, MessageCreateOptions } from 'discord.js'; +import { Message } from '../events'; import { Database } from '../shared/Database'; +import { LoggedMessage } from '../types'; import { channelIsInStaffCategory } from './channel-utils'; +import { deserializeMessagePart, getModifiedAttachments, getModifiedStickers } from './message-utils'; type CustomEventLogData = { title: string @@ -10,6 +13,63 @@ type CustomEventLogData = { fields: APIEmbedField[] | null } +export async function logEdit(loggedMessage: LoggedMessage, newMessage: Message) { + const channel = await getEventLogChannel(newMessage.guild, newMessage.channel); + + if (!channel) return; + + const includeContentInEmbed = loggedMessage.content.length <= 1024 && newMessage.content.length <= 1024; + + const fields: APIEmbedField[] = []; + fields.push(...getMainEmbeds(loggedMessage, newMessage)); + fields.push(...getEditEmbeds(loggedMessage, newMessage, includeContentInEmbed)); + + const embed = new EmbedBuilder() + .setTitle('Edited Message') + .setColor(0xFFFF00) + .setTimestamp(newMessage.createdTimestamp) + .addFields(...fields); + + const messagePayload: MessageCreateOptions = { embeds: [embed] }; + + if (!includeContentInEmbed) { + const before = new AttachmentBuilder(Buffer.from(loggedMessage.content), { name: 'before.txt' }); + const after = new AttachmentBuilder(Buffer.from(newMessage.content), { name: 'after.txt' }); + + messagePayload.files = [before, after]; + } + + channel.send(messagePayload); +} + +export async function logDeletion(loggedMessage: LoggedMessage, deletedMessage: Message) { + const channel = await getEventLogChannel(deletedMessage.guild, deletedMessage.channel); + + if (!channel) return; + + const includeContentInEmbed = loggedMessage.content.length <= 1024; + + const fields: APIEmbedField[] = []; + fields.push(...getMainEmbeds(loggedMessage, deletedMessage)); + fields.push(...getDeletedEmbeds(loggedMessage, includeContentInEmbed)); + + const embed = new EmbedBuilder() + .setTitle('Deleted Message') + .setColor(0xFF0000) + .setTimestamp(deletedMessage.createdTimestamp) + .addFields(...fields); + + const messagePayload: MessageCreateOptions = { embeds: [embed] }; + + if (!includeContentInEmbed) { + const before = new AttachmentBuilder(Buffer.from(loggedMessage.content), { name: 'content.txt' }); + + messagePayload.files = [before]; + } + + channel.send(messagePayload); +} + export async function logCustomEvent(guild: Guild, data: CustomEventLogData) { const channel = await getEventLogChannel(guild); @@ -47,4 +107,113 @@ async function getEventLogChannel(guild: Guild, channel: GuildBasedChannel | nul return channel; } +} + +function getMainEmbeds(loggedMessage: LoggedMessage, newMessage: Message): APIEmbedField[] { + const channelString = `${newMessage.channel.toString()}\n${newMessage.channel.name}`; + const userString = `<@${loggedMessage.author_id}>\n${loggedMessage.author_name}`; + + return [ + { + name: 'Channel', + value: channelString, + inline: true + }, + { + name: 'User', + value: userString, + inline: true + }, + { + name: 'Message', + value: `[${newMessage.id}](${newMessage.url})`, + inline: true + }, + ]; +} + +function getDeletedEmbeds(loggedMessage: LoggedMessage, includeContentInEmbed = true): APIEmbedField[] { + const fields: APIEmbedField[] = []; + + if (includeContentInEmbed && loggedMessage.content != '') { + fields.push({ + name: 'Content', + value: loggedMessage.content, + inline: false + }); + } + + for (const attachment of deserializeMessagePart(loggedMessage.attachments)) { + fields.push({ + name: 'Attachment', + value: attachment, + inline: true + }); + } + + for (const sticker of deserializeMessagePart(loggedMessage.stickers)) { + fields.push({ + name: 'Stickers', + value: sticker, + inline: true + }); + } + + return fields; +} + +function getEditEmbeds(loggedMessage: LoggedMessage, newMessage: Message, includeContentInEmbed = true): APIEmbedField[] { + const fields: APIEmbedField[] = []; + if (includeContentInEmbed && loggedMessage.content != newMessage.content) { + fields.push( + { + name: 'Before', + value: loggedMessage.content, + inline: false + }, + { + name: 'After', + value: newMessage.content, + inline: false + } + ); + } + + const { addedAttachments, removedAttachments } = getModifiedAttachments(loggedMessage, newMessage); + + for (const removedAttachment of removedAttachments) { + fields.push({ + name: 'Removed Attachment', + value: removedAttachment, + inline: true + }); + } + + for (const addedAttachment of addedAttachments) { + fields.push({ + name: 'Added Attachment', + value: addedAttachment, + inline: true + }); + } + + const { addedStickers, removedStickers } = getModifiedStickers(loggedMessage, newMessage); + + for (const removedSticker of addedStickers) { + fields.push({ + name: 'Removed Sticker', + value: removedSticker, + inline: true + }); + } + + for (const addedSticker of removedStickers) { + fields.push({ + name: 'Added Sticker', + value: addedSticker, + inline: true + }); + } + + return fields; } \ No newline at end of file diff --git a/src/utils/message-utils.ts b/src/utils/message-utils.ts index f1efe06..0a5837c 100644 --- a/src/utils/message-utils.ts +++ b/src/utils/message-utils.ts @@ -1,4 +1,5 @@ import { Message } from '../events'; +import { LoggedMessage } from '../types'; export const ARRAY_SEPARATOR = '$'; @@ -15,6 +16,34 @@ export function deserializeMessagePart(part: string): string[] { return part.split(ARRAY_SEPARATOR).filter(e => e); } +export function getModifiedAttachments(loggedMessage: LoggedMessage, newMessage: Message): { addedAttachments: string[], removedAttachments: string[] } { + const loggedAttachments = loggedMessage.attachments.split(ARRAY_SEPARATOR).filter(e => e); + const addedAttachments = newMessage.attachments.filter(a => !loggedAttachments.includes(`${a.name}:${a.id}`)).map(a => `${a.name}:${a.id}`); + const removedAttachments = loggedAttachments.filter(a => !newMessage.attachments.has(a.split(':').at(-1)!)); + + return { addedAttachments, removedAttachments }; +} + +export function getModifiedStickers(loggedMessage: LoggedMessage, newMessage: Message): { addedStickers: string[], removedStickers: string[] } { + const loggedStickers = loggedMessage.stickers.split(ARRAY_SEPARATOR).filter(e => e); + const addedStickers = newMessage.stickers.filter(s => !loggedStickers.includes(`${s.name}:${s.id}`)).map(s => `${s.name}:${s.id}`); + const removedStickers = loggedStickers.filter(s => !newMessage.stickers.has(s.split(':').at(-1)!)); + + return { addedStickers, removedStickers }; +} + +export function isEdited(loggedMessage: LoggedMessage, newMessage: Message) { + if (newMessage.content != loggedMessage.content) return true; + + const { addedAttachments, removedAttachments } = getModifiedAttachments(loggedMessage, newMessage); + + if (addedAttachments.length > 0 || removedAttachments.length > 0) return true; + + const { addedStickers, removedStickers } = getModifiedStickers(loggedMessage, newMessage); + + return addedStickers.length > 0 || removedStickers.length > 0; +} + export function isInSpoilerTags(content: string, index: number): boolean { if (!content.includes('||')) return false;