bump

2026-04-20 17:09:04 -05:00
parent 82f8a12e27
commit b83360db35
48 changed files with 6079 additions and 49 deletions
--- a/Bot/.gitignore
+++ b/Bot/.gitignore
@@ -0,0 +1,4 @@
+node_modules/
+data/
+config.json
+*.log
--- a/Bot/README.md
+++ b/Bot/README.md
@@ -0,0 +1,115 @@
+# Discord Backup Bot
+
+Archives a single Discord server's text channels in full fidelity: messages, attachments, embeds, reactions, reply/thread structure, edits, and deletions. Stores everything in a local SQLite database with attachment files on disk. Resumes cleanly after crashes or restarts.
+
+## Setup
+
+### 1. Create a Discord bot
+
+1. Open <https://discord.com/developers/applications> and create a new application.
+2. Under **Bot**, click **Reset Token** and copy the token somewhere safe.
+3. Under **Privileged Gateway Intents**, enable:
+   - **Message Content Intent** (required)
+4. Under **OAuth2 → URL Generator**:
+   - Scopes: `bot`
+   - Bot Permissions: `View Channels`, `Read Message History`
+5. Open the generated URL and add the bot to your server.
+
+### 2. Get the IDs
+
+Enable Developer Mode in Discord: **User Settings → Advanced → Developer Mode**.
+
+Right-click your server icon → **Copy Server ID** (this is `guildId`).
+Right-click each channel you want backed up → **Copy Channel ID**.
+
+### 3. Install and configure
+
+Requires Node.js 22.5 or newer (uses the built-in `node:sqlite` module — no native compilation).
+
+```bash
+npm install
+```
+
+Copy `config.example.json` to `config.json` and fill in:
+
+```json
+{
+  "token": "your bot token",
+  "guildId": "your server id",
+  "channelIds": ["channel id 1", "channel id 2"],
+  "dataDir": "./data",
+  "attachmentConcurrency": 3
+}
+```
+
+### 4. Run
+
+```bash
+npm start
+```
+
+The bot will log in, attach its live listener, start downloading attachments in the background, and backfill each configured channel (walking backward through old messages and forward to catch anything missed during downtime). Leave it running 24/7. Stop with Ctrl+C — next start resumes from the last checkpoint.
+
+## What gets captured
+
+- Messages (content, author, timestamps, reply target)
+- Edits — full edit history from when the bot is present
+- Deletions — soft-delete flag with timestamp
+- Attachments — downloaded to `data/attachments/<channel_id>/<message_id>/<filename>`
+- Embeds — link previews, bot embeds (stored as JSON)
+- Reactions — per user, with add/remove timestamps
+- Threads — active and public archived; new threads via live events
+- Author info — username, display name, avatar URL snapshot
+
+## Limitations
+
+- **Pre-bot edits and deletions are unrecoverable.** Discord's API does not expose edit/deletion history — the bot only sees them live.
+- **Reactions removed before the bot joined are lost.** Reactions still present at backfill time are captured.
+- **Private archived threads** are not backfilled (requires extra permissions). Private threads created while the bot is live are captured.
+- **Signed attachment URLs expire after ~24 hours.** If the bot is offline that long after a message is posted, its attachment URL may 403/410 before the downloader can fetch it — metadata is preserved but the file is unrecoverable.
+- **Gateway reconnects that don't trigger a fresh `ready`** don't re-run backfill. Restart the bot to catch up after long disconnects.
+
+## Storage
+
+```
+data/
+├── backup.db          # SQLite (WAL mode; -shm / -wal files appear while running)
+└── attachments/
+    └── <channel_id>/<message_id>/<attachment_id>/<filename>
+```
+
+## Inspecting the archive
+
+```bash
+sqlite3 data/backup.db
+```
+
+Useful queries:
+
+```sql
+-- Messages per channel (including threads)
+SELECT c.name, COUNT(*) AS count
+FROM messages m JOIN channels c ON c.id = m.channel_id
+GROUP BY c.id ORDER BY count DESC;
+
+-- Backfill progress
+SELECT name, is_thread, backfill_complete,
+       oldest_fetched_id, newest_fetched_id
+FROM channels;
+
+-- Pending / failed attachments
+SELECT
+  SUM(CASE WHEN downloaded = 1 THEN 1 ELSE 0 END) AS done,
+  SUM(CASE WHEN downloaded = 0 AND download_failed = 0 THEN 1 ELSE 0 END) AS pending,
+  SUM(download_failed) AS failed
+FROM attachments;
+
+-- Messages with edit history
+SELECT m.id, m.content, COUNT(e.message_id) AS edits
+FROM messages m LEFT JOIN message_edits e ON e.message_id = m.id
+GROUP BY m.id HAVING edits > 0;
+
+-- Deleted messages
+SELECT id, author_id, content, deleted_at FROM messages
+WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC LIMIT 20;
+```
--- a/Bot/config.example.json
+++ b/Bot/config.example.json
@@ -0,0 +1,7 @@
+{
+  "token": "YOUR_BOT_TOKEN",
+  "guildId": "YOUR_GUILD_ID",
+  "channelIds": ["CHANNEL_ID_1", "CHANNEL_ID_2"],
+  "dataDir": "./data",
+  "attachmentConcurrency": 3
+}
--- a/Bot/package-lock.json
+++ b/Bot/package-lock.json
@@ -0,0 +1,327 @@
+{
+  "name": "discord-backup-bot",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "discord-backup-bot",
+      "version": "1.0.0",
+      "dependencies": {
+        "discord.js": "^14.16.3"
+      },
+      "engines": {
+        "node": ">=22.5"
+      }
+    },
+    "node_modules/@discordjs/builders": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.14.1.tgz",
+      "integrity": "sha512-gSKkhXLqs96TCzk66VZuHHl8z2bQMJFGwrXC0f33ngK+FLNau4hU1PYny3DNJfNdSH+gVMzE85/d5FQ2BpcNwQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@discordjs/formatters": "^0.6.2",
+        "@discordjs/util": "^1.2.0",
+        "@sapphire/shapeshift": "^4.0.0",
+        "discord-api-types": "^0.38.40",
+        "fast-deep-equal": "^3.1.3",
+        "ts-mixer": "^6.0.4",
+        "tslib": "^2.6.3"
+      },
+      "engines": {
+        "node": ">=16.11.0"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/collection": {
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.5.3.tgz",
+      "integrity": "sha512-SVb428OMd3WO1paV3rm6tSjM4wC+Kecaa1EUGX7vc6/fddvw/6lg90z4QtCqm21zvVe92vMMDt9+DkIvjXImQQ==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=16.11.0"
+      }
+    },
+    "node_modules/@discordjs/formatters": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/@discordjs/formatters/-/formatters-0.6.2.tgz",
+      "integrity": "sha512-y4UPwWhH6vChKRkGdMB4odasUbHOUwy7KL+OVwF86PvT6QVOwElx+TiI1/6kcmcEe+g5YRXJFiXSXUdabqZOvQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "discord-api-types": "^0.38.33"
+      },
+      "engines": {
+        "node": ">=16.11.0"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/rest": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-2.6.1.tgz",
+      "integrity": "sha512-wwQdgjeaoYFiaG+atbqx6aJDpqW7JHAo0HrQkBTbYzM3/PJ3GweQIpgElNcGZ26DCUOXMyawYd0YF7vtr+fZXg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@discordjs/collection": "^2.1.1",
+        "@discordjs/util": "^1.2.0",
+        "@sapphire/async-queue": "^1.5.3",
+        "@sapphire/snowflake": "^3.5.5",
+        "@vladfrangu/async_event_emitter": "^2.4.6",
+        "discord-api-types": "^0.38.40",
+        "magic-bytes.js": "^1.13.0",
+        "tslib": "^2.6.3",
+        "undici": "6.24.1"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/rest/node_modules/@discordjs/collection": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-2.1.1.tgz",
+      "integrity": "sha512-LiSusze9Tc7qF03sLCujF5iZp7K+vRNEDBZ86FT9aQAv3vxMLihUvKvpsCWiQ2DJq1tVckopKm1rxomgNUc9hg==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/rest/node_modules/@sapphire/snowflake": {
+      "version": "3.5.5",
+      "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.5.5.tgz",
+      "integrity": "sha512-xzvBr1Q1c4lCe7i6sRnrofxeO1QTP/LKQ6A6qy0iB4x5yfiSfARMEQEghojzTNALDTcv8En04qYNIco9/K9eZQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=v14.0.0",
+        "npm": ">=7.0.0"
+      }
+    },
+    "node_modules/@discordjs/util": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@discordjs/util/-/util-1.2.0.tgz",
+      "integrity": "sha512-3LKP7F2+atl9vJFhaBjn4nOaSWahZ/yWjOvA4e5pnXkt2qyXRCHLxoBQy81GFtLGCq7K9lPm9R517M1U+/90Qg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "discord-api-types": "^0.38.33"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/ws": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/@discordjs/ws/-/ws-1.2.3.tgz",
+      "integrity": "sha512-wPlQDxEmlDg5IxhJPuxXr3Vy9AjYq5xCvFWGJyD7w7Np8ZGu+Mc+97LCoEc/+AYCo2IDpKioiH0/c/mj5ZR9Uw==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@discordjs/collection": "^2.1.0",
+        "@discordjs/rest": "^2.5.1",
+        "@discordjs/util": "^1.1.0",
+        "@sapphire/async-queue": "^1.5.2",
+        "@types/ws": "^8.5.10",
+        "@vladfrangu/async_event_emitter": "^2.2.4",
+        "discord-api-types": "^0.38.1",
+        "tslib": "^2.6.2",
+        "ws": "^8.17.0"
+      },
+      "engines": {
+        "node": ">=16.11.0"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@discordjs/ws/node_modules/@discordjs/collection": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-2.1.1.tgz",
+      "integrity": "sha512-LiSusze9Tc7qF03sLCujF5iZp7K+vRNEDBZ86FT9aQAv3vxMLihUvKvpsCWiQ2DJq1tVckopKm1rxomgNUc9hg==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/@sapphire/async-queue": {
+      "version": "1.5.5",
+      "resolved": "https://registry.npmjs.org/@sapphire/async-queue/-/async-queue-1.5.5.tgz",
+      "integrity": "sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=v14.0.0",
+        "npm": ">=7.0.0"
+      }
+    },
+    "node_modules/@sapphire/shapeshift": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-4.0.0.tgz",
+      "integrity": "sha512-d9dUmWVA7MMiKobL3VpLF8P2aeanRTu6ypG2OIaEv/ZHH/SUQ2iHOVyi5wAPjQ+HmnMuL0whK9ez8I/raWbtIg==",
+      "license": "MIT",
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3",
+        "lodash": "^4.17.21"
+      },
+      "engines": {
+        "node": ">=v16"
+      }
+    },
+    "node_modules/@sapphire/snowflake": {
+      "version": "3.5.3",
+      "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.5.3.tgz",
+      "integrity": "sha512-jjmJywLAFoWeBi1W7994zZyiNWPIiqRRNAmSERxyg93xRGzNYvGjlZ0gR6x0F4gPRi2+0O6S71kOZYyr3cxaIQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=v14.0.0",
+        "npm": ">=7.0.0"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "25.6.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz",
+      "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~7.19.0"
+      }
+    },
+    "node_modules/@types/ws": {
+      "version": "8.18.1",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
+      "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@vladfrangu/async_event_emitter": {
+      "version": "2.4.7",
+      "resolved": "https://registry.npmjs.org/@vladfrangu/async_event_emitter/-/async_event_emitter-2.4.7.tgz",
+      "integrity": "sha512-Xfe6rpCTxSxfbswi/W/Pz7zp1WWSNn4A0eW4mLkQUewCrXXtMj31lCg+iQyTkh/CkusZSq9eDflu7tjEDXUY6g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=v14.0.0",
+        "npm": ">=7.0.0"
+      }
+    },
+    "node_modules/discord-api-types": {
+      "version": "0.38.47",
+      "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.47.tgz",
+      "integrity": "sha512-XgXQodHQBAE6kfD7kMvVo30863iHX1LHSqNq6MGUTDwIFCCvHva13+rwxyxVXDqudyApMNAd32PGjgVETi5rjA==",
+      "license": "MIT",
+      "workspaces": [
+        "scripts/actions/documentation"
+      ]
+    },
+    "node_modules/discord.js": {
+      "version": "14.26.3",
+      "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.26.3.tgz",
+      "integrity": "sha512-XEKtYn28YFsiJ5l4fLRyikdbo6RD5oFyqfVHQlvXz2104JhH/E8slN28dbky05w3DCrJcNVWvhVvcJCTSl/KIg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@discordjs/builders": "^1.14.1",
+        "@discordjs/collection": "1.5.3",
+        "@discordjs/formatters": "^0.6.2",
+        "@discordjs/rest": "^2.6.1",
+        "@discordjs/util": "^1.2.0",
+        "@discordjs/ws": "^1.2.3",
+        "@sapphire/snowflake": "3.5.3",
+        "discord-api-types": "^0.38.40",
+        "fast-deep-equal": "3.1.3",
+        "lodash.snakecase": "4.1.1",
+        "magic-bytes.js": "^1.13.0",
+        "tslib": "^2.6.3",
+        "undici": "6.24.1"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/discordjs/discord.js?sponsor"
+      }
+    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "license": "MIT"
+    },
+    "node_modules/lodash": {
+      "version": "4.18.1",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
+      "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.snakecase": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz",
+      "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==",
+      "license": "MIT"
+    },
+    "node_modules/magic-bytes.js": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/magic-bytes.js/-/magic-bytes.js-1.13.0.tgz",
+      "integrity": "sha512-afO2mnxW7GDTXMm5/AoN1WuOcdoKhtgXjIvHmobqTD1grNplhGdv3PFOyjCVmrnOZBIT/gD/koDKpYG+0mvHcg==",
+      "license": "MIT"
+    },
+    "node_modules/ts-mixer": {
+      "version": "6.0.4",
+      "resolved": "https://registry.npmjs.org/ts-mixer/-/ts-mixer-6.0.4.tgz",
+      "integrity": "sha512-ufKpbmrugz5Aou4wcr5Wc1UUFWOLhq+Fm6qa6P0w0K5Qw2yhaUoiWszhCVuNQyNwrlGiscHOmqYoAox1PtvgjA==",
+      "license": "MIT"
+    },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD"
+    },
+    "node_modules/undici": {
+      "version": "6.24.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.1.tgz",
+      "integrity": "sha512-sC+b0tB1whOCzbtlx20fx3WgCXwkW627p4EA9uM+/tNNPkSS+eSEld6pAs9nDv7WbY1UUljBMYPtu9BCOrCWKA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "7.19.2",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz",
+      "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==",
+      "license": "MIT"
+    },
+    "node_modules/ws": {
+      "version": "8.20.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz",
+      "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  }
+}
--- a/Bot/package.json
+++ b/Bot/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "discord-backup-bot",
+  "version": "1.0.0",
+  "private": true,
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node --no-warnings=ExperimentalWarning src/index.js"
+  },
+  "dependencies": {
+    "discord.js": "^14.16.3"
+  },
+  "engines": {
+    "node": ">=22.5"
+  }
+}
--- a/Bot/src/backfill.js
+++ b/Bot/src/backfill.js
@@ -0,0 +1,132 @@
+const { writeMessageCore, emojiKey, updateNewestIfGreater } = require('./writers');
+
+async function backfillTarget(db, stmts, target) {
+  stmts.insertChannel.run(
+    target.id,
+    target.name ?? null,
+    target.parentId ?? null,
+    target.isThread?.() ? 1 : 0,
+  );
+
+  let row = stmts.getChannel.get(target.id);
+  if (row.newest_fetched_id) {
+    await walkForward(db, stmts, target, row.newest_fetched_id);
+  }
+
+  row = stmts.getChannel.get(target.id);
+  if (!row.backfill_complete) {
+    await walkBackward(db, stmts, target, row.oldest_fetched_id);
+    stmts.markBackfillComplete.run(target.id);
+  }
+}
+
+async function backfillChannelAndThreads(db, stmts, channel) {
+  console.log(`[backfill] channel ${channel.name} (${channel.id})`);
+  await backfillTarget(db, stmts, channel);
+
+  if (typeof channel.threads?.fetchActive === 'function') {
+    try {
+      const active = await channel.threads.fetchActive();
+      for (const thread of active.threads.values()) {
+        console.log(`[backfill]   active thread ${thread.name} (${thread.id})`);
+        await backfillTarget(db, stmts, thread);
+      }
+    } catch (err) {
+      console.error(`[backfill] active threads for ${channel.id}:`, err.message);
+    }
+
+    try {
+      let before;
+      while (true) {
+        const batch = await channel.threads.fetchArchived({ type: 'public', before, limit: 100 });
+        for (const thread of batch.threads.values()) {
+          console.log(`[backfill]   archived thread ${thread.name} (${thread.id})`);
+          stmts.updateArchived.run(1, thread.id);
+          await backfillTarget(db, stmts, thread);
+        }
+        if (!batch.hasMore) break;
+        const threadArr = [...batch.threads.values()];
+        const last = threadArr[threadArr.length - 1];
+        before = last.archivedAt ?? last.archiveTimestamp;
+        if (!before) break;
+      }
+    } catch (err) {
+      console.error(`[backfill] archived threads for ${channel.id}:`, err.message);
+    }
+  }
+}
+
+async function walkForward(db, stmts, target, afterId) {
+  let cursor = afterId;
+  while (true) {
+    const batch = await target.messages.fetch({ limit: 100, after: cursor, cache: false });
+    if (batch.size === 0) return;
+    const messages = [...batch.values()].sort((a, b) =>
+      BigInt(a.id) < BigInt(b.id) ? -1 : 1
+    );
+    await writeBatch(db, stmts, target, messages);
+    cursor = messages[messages.length - 1].id;
+    updateNewestIfGreater(stmts, target.id, cursor);
+  }
+}
+
+async function walkBackward(db, stmts, target, beforeId) {
+  let cursor = beforeId;
+  let emptyRuns = 0;
+  while (true) {
+    const opts = { limit: 100, cache: false };
+    if (cursor) opts.before = cursor;
+    const batch = await target.messages.fetch(opts);
+    if (batch.size === 0) {
+      emptyRuns++;
+      if (emptyRuns >= 2) return;
+      console.log(`[backfill] empty batch for ${target.id}, retrying once after 3s`);
+      await new Promise((r) => setTimeout(r, 3000));
+      continue;
+    }
+    emptyRuns = 0;
+    const messages = [...batch.values()].sort((a, b) =>
+      BigInt(a.id) < BigInt(b.id) ? 1 : -1
+    );
+    await writeBatch(db, stmts, target, messages);
+
+    const newestInBatch = messages[0].id;
+    const oldestInBatch = messages[messages.length - 1].id;
+    updateNewestIfGreater(stmts, target.id, newestInBatch);
+    stmts.updateOldest.run(oldestInBatch, target.id);
+    cursor = oldestInBatch;
+  }
+}
+
+async function writeBatch(db, stmts, target, messages) {
+  const reactionData = [];
+  for (const msg of messages) {
+    if (!msg.reactions?.cache) continue;
+    for (const reaction of msg.reactions.cache.values()) {
+      try {
+        const users = await reaction.users.fetch();
+        reactionData.push({
+          messageId: msg.id,
+          emoji: emojiKey(reaction.emoji),
+          userIds: [...users.keys()],
+        });
+      } catch (err) {
+        console.error(`[backfill] fetch reactions for ${msg.id}:`, err.message);
+      }
+    }
+  }
+
+  const tx = db.transaction(() => {
+    for (const msg of messages) {
+      writeMessageCore(stmts, msg);
+    }
+    for (const r of reactionData) {
+      for (const userId of r.userIds) {
+        stmts.upsertReaction.run(r.messageId, r.emoji, userId, null);
+      }
+    }
+  });
+  tx();
+}
+
+module.exports = { backfillChannelAndThreads };
--- a/Bot/src/db.js
+++ b/Bot/src/db.js
@@ -0,0 +1,30 @@
+const { DatabaseSync } = require('node:sqlite');
+const path = require('node:path');
+const fs = require('node:fs');
+
+function openDb(dataDir) {
+  fs.mkdirSync(dataDir, { recursive: true });
+  const db = new DatabaseSync(path.join(dataDir, 'backup.db'));
+  db.exec('PRAGMA journal_mode = WAL');
+  db.exec('PRAGMA synchronous = NORMAL');
+  db.exec('PRAGMA foreign_keys = ON');
+
+  const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf8');
+  db.exec(schema);
+
+  db.transaction = (fn) => (...args) => {
+    db.exec('BEGIN');
+    try {
+      const result = fn(...args);
+      db.exec('COMMIT');
+      return result;
+    } catch (err) {
+      try { db.exec('ROLLBACK'); } catch (_) { /* nested */ }
+      throw err;
+    }
+  };
+
+  return db;
+}
+
+module.exports = { openDb };
--- a/Bot/src/discord.js
+++ b/Bot/src/discord.js
@@ -0,0 +1,15 @@
+const { Client, GatewayIntentBits, Partials } = require('discord.js');
+
+function createClient() {
+  return new Client({
+    intents: [
+      GatewayIntentBits.Guilds,
+      GatewayIntentBits.GuildMessages,
+      GatewayIntentBits.MessageContent,
+      GatewayIntentBits.GuildMessageReactions,
+    ],
+    partials: [Partials.Message, Partials.Channel, Partials.Reaction],
+  });
+}
+
+module.exports = { createClient };
--- a/Bot/src/downloader.js
+++ b/Bot/src/downloader.js
@@ -0,0 +1,69 @@
+const path = require('node:path');
+const fs = require('node:fs');
+const { Readable } = require('node:stream');
+const { pipeline } = require('node:stream/promises');
+
+async function runDownloader(stmts, dataDir, concurrency, getRunning) {
+  const attachmentsDir = path.join(dataDir, 'attachments');
+  fs.mkdirSync(attachmentsDir, { recursive: true });
+
+  while (getRunning()) {
+    const pending = stmts.pendingAttachments.all(concurrency * 4);
+    if (pending.length === 0) {
+      await sleep(5000);
+      continue;
+    }
+
+    const queue = [...pending];
+    const workers = Array.from({ length: concurrency }, () =>
+      worker(queue, stmts, attachmentsDir, dataDir, getRunning)
+    );
+    await Promise.all(workers);
+  }
+}
+
+async function worker(queue, stmts, attachmentsDir, dataDir, getRunning) {
+  while (queue.length > 0 && getRunning()) {
+    const item = queue.shift();
+    if (!item) return;
+    await downloadOne(item, stmts, attachmentsDir, dataDir);
+  }
+}
+
+async function downloadOne(item, stmts, attachmentsDir, dataDir) {
+  const destDir = path.join(attachmentsDir, item.channel_id, item.message_id, item.id);
+  const destPath = path.join(destDir, safeFilename(item.filename));
+
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      fs.mkdirSync(destDir, { recursive: true });
+      const res = await fetch(item.url);
+      if (res.status === 404 || res.status === 403 || res.status === 410) {
+        console.error(`[download] ${item.filename} → HTTP ${res.status} (permanent)`);
+        stmts.markAttachmentFailed.run(item.id);
+        return;
+      }
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const tmpPath = destPath + '.part';
+      await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmpPath));
+      fs.renameSync(tmpPath, destPath);
+      const relPath = path.relative(dataDir, destPath);
+      stmts.markAttachmentDownloaded.run(relPath, item.id);
+      return;
+    } catch (err) {
+      console.error(`[download] ${item.filename} attempt ${attempt + 1}: ${err.message}`);
+      if (attempt < 2) await sleep(1000 * (attempt + 1));
+    }
+  }
+  stmts.markAttachmentFailed.run(item.id);
+}
+
+function safeFilename(name) {
+  return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_').slice(0, 200);
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+module.exports = { runDownloader };
--- a/Bot/src/index.js
+++ b/Bot/src/index.js
@@ -0,0 +1,80 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const { openDb } = require('./db');
+const { prepareStatements } = require('./writers');
+const { createClient } = require('./discord');
+const { backfillChannelAndThreads } = require('./backfill');
+const { attachHandlers } = require('./live');
+const { runDownloader } = require('./downloader');
+
+function loadConfig() {
+  const configPath = path.resolve(process.cwd(), 'config.json');
+  if (!fs.existsSync(configPath)) {
+    throw new Error(`config.json not found at ${configPath}. Copy config.example.json and fill it in.`);
+  }
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+  if (!config.token) throw new Error('config.json missing "token"');
+  if (!config.guildId) throw new Error('config.json missing "guildId"');
+  if (!Array.isArray(config.channelIds) || config.channelIds.length === 0) {
+    throw new Error('config.json must have a non-empty "channelIds" array');
+  }
+  return config;
+}
+
+async function main() {
+  const config = loadConfig();
+  const dataDir = path.resolve(process.cwd(), config.dataDir ?? './data');
+  const db = openDb(dataDir);
+  const stmts = prepareStatements(db);
+  const client = createClient();
+
+  attachHandlers(client, db, stmts, config.channelIds);
+
+  let running = true;
+  const shutdown = () => {
+    if (!running) return;
+    console.log('[bot] shutting down');
+    running = false;
+    client.destroy();
+    try {
+      db.close();
+    } catch (err) {
+      console.error('[bot] db close:', err.message);
+    }
+    setTimeout(() => process.exit(0), 500).unref();
+  };
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+
+  client.once('ready', async (c) => {
+    console.log(`[bot] logged in as ${c.user.tag}`);
+
+    runDownloader(stmts, dataDir, config.attachmentConcurrency ?? 3, () => running).catch((err) => {
+      console.error('[downloader] fatal:', err);
+    });
+
+    const guild = await c.guilds.fetch(config.guildId);
+    for (const channelId of config.channelIds) {
+      if (!running) return;
+      try {
+        const channel = await guild.channels.fetch(channelId);
+        if (!channel) {
+          console.error(`[backfill] channel ${channelId} not found or not accessible`);
+          continue;
+        }
+        await backfillChannelAndThreads(db, stmts, channel);
+        console.log(`[backfill] complete for ${channel.name}`);
+      } catch (err) {
+        console.error(`[backfill] ${channelId}:`, err.message);
+      }
+    }
+    console.log('[backfill] all channels complete; live listener active');
+  });
+
+  await client.login(config.token);
+}
+
+main().catch((err) => {
+  console.error('fatal:', err);
+  process.exit(1);
+});
--- a/Bot/src/live.js
+++ b/Bot/src/live.js
@@ -0,0 +1,153 @@
+const { Events } = require('discord.js');
+const { writeMessageCore, emojiKey, updateNewestIfGreater } = require('./writers');
+
+function attachHandlers(client, db, stmts, channelIds) {
+  const channelSet = new Set(channelIds);
+
+  const parentOf = (channel) => (channel?.isThread?.() ? channel.parentId : null);
+  const inScope = (channelId, parentId) =>
+    channelSet.has(channelId) || (parentId && channelSet.has(parentId));
+
+  client.on(Events.MessageCreate, (msg) => {
+    const parentId = parentOf(msg.channel);
+    if (!inScope(msg.channelId, parentId)) return;
+    const tx = db.transaction(() => {
+      if (msg.channel?.isThread?.()) {
+        stmts.insertChannel.run(msg.channel.id, msg.channel.name, msg.channel.parentId, 1);
+      }
+      writeMessageCore(stmts, msg);
+      updateNewestIfGreater(stmts, msg.channelId, msg.id);
+    });
+    try {
+      tx();
+    } catch (err) {
+      console.error('[live] messageCreate:', err.message);
+    }
+  });
+
+  client.on(Events.MessageUpdate, (_oldMsg, newMsg) => {
+    const parentId = parentOf(newMsg.channel);
+    if (!inScope(newMsg.channelId, parentId)) return;
+    const editedAt = newMsg.editedTimestamp ?? Date.now();
+    const tx = db.transaction(() => {
+      const existing = stmts.existingContent.get(newMsg.id);
+      if (existing) {
+        stmts.insertEdit.run(newMsg.id, existing.content, editedAt);
+        stmts.updateContent.run(newMsg.content ?? '', editedAt, newMsg.id);
+      } else {
+        writeMessageCore(stmts, newMsg);
+      }
+    });
+    try {
+      tx();
+    } catch (err) {
+      console.error('[live] messageUpdate:', err.message);
+    }
+  });
+
+  client.on(Events.MessageDelete, (msg) => {
+    const parentId = parentOf(msg.channel);
+    if (!inScope(msg.channelId, parentId)) return;
+    const now = Date.now();
+    const tx = db.transaction(() => {
+      stmts.insertDeletedPlaceholder.run(
+        msg.id,
+        msg.channelId,
+        msg.author?.id ?? null,
+        msg.content ?? '',
+        msg.createdTimestamp ?? now,
+        now,
+      );
+      stmts.markDeleted.run(now, msg.id);
+    });
+    try {
+      tx();
+    } catch (err) {
+      console.error('[live] messageDelete:', err.message);
+    }
+  });
+
+  client.on(Events.MessageBulkDelete, (messages) => {
+    const now = Date.now();
+    const tx = db.transaction(() => {
+      for (const msg of messages.values()) {
+        const parentId = parentOf(msg.channel);
+        if (!inScope(msg.channelId, parentId)) continue;
+        stmts.insertDeletedPlaceholder.run(
+          msg.id,
+          msg.channelId,
+          msg.author?.id ?? null,
+          msg.content ?? '',
+          msg.createdTimestamp ?? now,
+          now,
+        );
+        stmts.markDeleted.run(now, msg.id);
+      }
+    });
+    try {
+      tx();
+    } catch (err) {
+      console.error('[live] messageDeleteBulk:', err.message);
+    }
+  });
+
+  client.on(Events.MessageReactionAdd, (reaction, user) => {
+    const parentId = parentOf(reaction.message.channel);
+    if (!inScope(reaction.message.channelId, parentId)) return;
+    try {
+      stmts.upsertReaction.run(reaction.message.id, emojiKey(reaction.emoji), user.id, Date.now());
+    } catch (err) {
+      console.error('[live] reactionAdd:', err.message);
+    }
+  });
+
+  client.on(Events.MessageReactionRemove, (reaction, user) => {
+    const parentId = parentOf(reaction.message.channel);
+    if (!inScope(reaction.message.channelId, parentId)) return;
+    try {
+      stmts.removeReaction.run(Date.now(), reaction.message.id, emojiKey(reaction.emoji), user.id);
+    } catch (err) {
+      console.error('[live] reactionRemove:', err.message);
+    }
+  });
+
+  client.on(Events.MessageReactionRemoveAll, (msg) => {
+    const parentId = parentOf(msg.channel);
+    if (!inScope(msg.channelId, parentId)) return;
+    try {
+      stmts.removeAllReactions.run(Date.now(), msg.id);
+    } catch (err) {
+      console.error('[live] reactionRemoveAll:', err.message);
+    }
+  });
+
+  client.on(Events.ThreadCreate, (thread) => {
+    if (!channelSet.has(thread.parentId)) return;
+    try {
+      stmts.insertChannel.run(thread.id, thread.name, thread.parentId, 1);
+    } catch (err) {
+      console.error('[live] threadCreate:', err.message);
+    }
+  });
+
+  client.on(Events.ThreadUpdate, (_oldT, newT) => {
+    if (!channelSet.has(newT.parentId)) return;
+    try {
+      stmts.insertChannel.run(newT.id, newT.name, newT.parentId, 1);
+      stmts.updateArchived.run(newT.archived ? 1 : 0, newT.id);
+    } catch (err) {
+      console.error('[live] threadUpdate:', err.message);
+    }
+  });
+
+  client.on(Events.ThreadDelete, (thread) => {
+    if (!channelSet.has(thread.parentId)) return;
+    try {
+      stmts.updateArchived.run(1, thread.id);
+    } catch (err) {
+      console.error('[live] threadDelete:', err.message);
+    }
+  });
+}
+
+module.exports = { attachHandlers };
--- a/Bot/src/schema.sql
+++ b/Bot/src/schema.sql
@@ -0,0 +1,68 @@
+CREATE TABLE IF NOT EXISTS channels (
+  id TEXT PRIMARY KEY,
+  name TEXT,
+  parent_channel_id TEXT,
+  is_thread INTEGER NOT NULL DEFAULT 0,
+  archived INTEGER NOT NULL DEFAULT 0,
+  oldest_fetched_id TEXT,
+  newest_fetched_id TEXT,
+  backfill_complete INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS messages (
+  id TEXT PRIMARY KEY,
+  channel_id TEXT NOT NULL,
+  author_id TEXT,
+  content TEXT,
+  created_at INTEGER NOT NULL,
+  edited_at INTEGER,
+  deleted_at INTEGER,
+  replied_to_id TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_messages_channel_created ON messages(channel_id, created_at);
+
+CREATE TABLE IF NOT EXISTS message_edits (
+  message_id TEXT NOT NULL,
+  content TEXT,
+  edited_at INTEGER NOT NULL,
+  PRIMARY KEY (message_id, edited_at)
+);
+
+CREATE TABLE IF NOT EXISTS attachments (
+  id TEXT PRIMARY KEY,
+  message_id TEXT NOT NULL,
+  filename TEXT NOT NULL,
+  url TEXT NOT NULL,
+  local_path TEXT,
+  size INTEGER,
+  content_type TEXT,
+  downloaded INTEGER NOT NULL DEFAULT 0,
+  download_failed INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_attachments_message ON attachments(message_id);
+CREATE INDEX IF NOT EXISTS idx_attachments_pending ON attachments(downloaded)
+  WHERE downloaded = 0 AND download_failed = 0;
+
+CREATE TABLE IF NOT EXISTS reactions (
+  message_id TEXT NOT NULL,
+  emoji TEXT NOT NULL,
+  user_id TEXT NOT NULL,
+  added_at INTEGER,
+  removed_at INTEGER,
+  PRIMARY KEY (message_id, emoji, user_id)
+);
+
+CREATE TABLE IF NOT EXISTS embeds (
+  message_id TEXT NOT NULL,
+  idx INTEGER NOT NULL,
+  data_json TEXT NOT NULL,
+  PRIMARY KEY (message_id, idx)
+);
+
+CREATE TABLE IF NOT EXISTS authors (
+  id TEXT PRIMARY KEY,
+  username TEXT,
+  display_name TEXT,
+  avatar_url TEXT,
+  last_seen INTEGER
+);
--- a/Bot/src/writers.js
+++ b/Bot/src/writers.js
@@ -0,0 +1,142 @@
+function prepareStatements(db) {
+  return {
+    insertChannel: db.prepare(`
+      INSERT INTO channels (id, name, parent_channel_id, is_thread)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        name = excluded.name,
+        parent_channel_id = excluded.parent_channel_id,
+        is_thread = excluded.is_thread
+    `),
+    updateArchived: db.prepare('UPDATE channels SET archived = ? WHERE id = ?'),
+    updateOldest: db.prepare('UPDATE channels SET oldest_fetched_id = ? WHERE id = ?'),
+    updateNewest: db.prepare(`
+      UPDATE channels
+      SET newest_fetched_id = CASE
+        WHEN newest_fetched_id IS NULL THEN ?
+        WHEN CAST(? AS INTEGER) > CAST(newest_fetched_id AS INTEGER) THEN ?
+        ELSE newest_fetched_id
+      END
+      WHERE id = ?
+    `),
+    markBackfillComplete: db.prepare('UPDATE channels SET backfill_complete = 1 WHERE id = ?'),
+    getChannel: db.prepare('SELECT * FROM channels WHERE id = ?'),
+
+    insertMessage: db.prepare(`
+      INSERT OR IGNORE INTO messages
+        (id, channel_id, author_id, content, created_at, edited_at, replied_to_id)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `),
+    insertDeletedPlaceholder: db.prepare(`
+      INSERT OR IGNORE INTO messages
+        (id, channel_id, author_id, content, created_at, deleted_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `),
+    markDeleted: db.prepare('UPDATE messages SET deleted_at = ? WHERE id = ? AND deleted_at IS NULL'),
+    updateContent: db.prepare('UPDATE messages SET content = ?, edited_at = ? WHERE id = ?'),
+    existingContent: db.prepare('SELECT content FROM messages WHERE id = ?'),
+    insertEdit: db.prepare(`
+      INSERT OR IGNORE INTO message_edits (message_id, content, edited_at) VALUES (?, ?, ?)
+    `),
+
+    insertAttachment: db.prepare(`
+      INSERT OR IGNORE INTO attachments (id, message_id, filename, url, size, content_type)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `),
+    markAttachmentDownloaded: db.prepare(
+      'UPDATE attachments SET downloaded = 1, local_path = ? WHERE id = ?'
+    ),
+    markAttachmentFailed: db.prepare('UPDATE attachments SET download_failed = 1 WHERE id = ?'),
+    pendingAttachments: db.prepare(`
+      SELECT a.id, a.message_id, a.filename, a.url, m.channel_id
+      FROM attachments a JOIN messages m ON m.id = a.message_id
+      WHERE a.downloaded = 0 AND a.download_failed = 0
+      LIMIT ?
+    `),
+
+    upsertReaction: db.prepare(`
+      INSERT INTO reactions (message_id, emoji, user_id, added_at)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(message_id, emoji, user_id) DO UPDATE SET
+        added_at = COALESCE(reactions.added_at, excluded.added_at),
+        removed_at = NULL
+    `),
+    removeReaction: db.prepare(`
+      UPDATE reactions SET removed_at = ?
+      WHERE message_id = ? AND emoji = ? AND user_id = ? AND removed_at IS NULL
+    `),
+    removeAllReactions: db.prepare(`
+      UPDATE reactions SET removed_at = ?
+      WHERE message_id = ? AND removed_at IS NULL
+    `),
+
+    insertEmbed: db.prepare(`
+      INSERT OR REPLACE INTO embeds (message_id, idx, data_json) VALUES (?, ?, ?)
+    `),
+
+    upsertAuthor: db.prepare(`
+      INSERT INTO authors (id, username, display_name, avatar_url, last_seen)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        username = excluded.username,
+        display_name = excluded.display_name,
+        avatar_url = excluded.avatar_url,
+        last_seen = MAX(COALESCE(authors.last_seen, 0), COALESCE(excluded.last_seen, 0))
+    `),
+  };
+}
+
+function emojiKey(emoji) {
+  if (emoji.id) {
+    return `<${emoji.animated ? 'a' : ''}:${emoji.name}:${emoji.id}>`;
+  }
+  return emoji.name;
+}
+
+function writeMessageCore(stmts, msg) {
+  const createdAt = msg.createdTimestamp;
+  const editedAt = msg.editedTimestamp ?? null;
+
+  if (msg.author) {
+    stmts.upsertAuthor.run(
+      msg.author.id,
+      msg.author.username ?? null,
+      msg.member?.displayName ?? msg.author.globalName ?? null,
+      msg.author.displayAvatarURL?.() ?? null,
+      createdAt,
+    );
+  }
+
+  stmts.insertMessage.run(
+    msg.id,
+    msg.channelId,
+    msg.author?.id ?? null,
+    msg.content ?? '',
+    createdAt,
+    editedAt,
+    msg.reference?.messageId ?? null,
+  );
+
+  for (const att of msg.attachments.values()) {
+    stmts.insertAttachment.run(
+      att.id,
+      msg.id,
+      att.name,
+      att.url,
+      att.size ?? null,
+      att.contentType ?? null,
+    );
+  }
+
+  let idx = 0;
+  for (const embed of msg.embeds) {
+    const data = typeof embed.toJSON === 'function' ? embed.toJSON() : embed;
+    stmts.insertEmbed.run(msg.id, idx++, JSON.stringify(data));
+  }
+}
+
+function updateNewestIfGreater(stmts, channelId, messageId) {
+  stmts.updateNewest.run(messageId, messageId, messageId, channelId);
+}
+
+module.exports = { prepareStatements, emojiKey, writeMessageCore, updateNewestIfGreater };