i hyperfocused and here is v1

This commit is contained in:
Rose 2024-04-19 00:15:20 -04:00
parent 8f43e6a6a9
commit 4090fd621e
No known key found for this signature in database
34 changed files with 4135 additions and 22 deletions

View file

@ -1 +1,3 @@
node_modules
.env
dist

6
.gitignore vendored
View file

@ -13,6 +13,10 @@
.pnp.*
build/
dist/
node_modules
.turbo
.idea
.env
packages/database/migrations

38
apps/bot/Dockerfile Normal file
View file

@ -0,0 +1,38 @@
FROM node:21-alpine AS base
FROM base AS builder
RUN apk add --no-cache libc6-compat
RUN apk update
RUN yarn set version canary
RUN yarn config set nodeLinker node-modules
WORKDIR /app
COPY . .
RUN yarn dlx turbo prune @datamine/bot --docker
FROM base AS installer
RUN apk add --no-cache libc6-compat
RUN apk update
WORKDIR /app
COPY .gitignore .gitignore
COPY --from=builder /app/out/json/ .
COPY --from=builder /app/out/yarn.lock ./yarn.lock
RUN yarn install
COPY --from=builder /app/out/full/ .
COPY turbo.json turbo.json
RUN yarn turbo build --filter=bot...
RUN yarn workspace @datamine/database run drizzle
FROM base AS runner
WORKDIR /app
# Don't run production as root
RUN addgroup --system --gid 1001 datamine
RUN adduser --system --uid 1001 datamine
USER datamine
COPY --from=installer /app .
CMD node apps/bot/dist/main.js

1
apps/bot/README.md Normal file
View file

@ -0,0 +1 @@
# @datamine/bot

4
apps/bot/biome.json Normal file
View file

@ -0,0 +1,4 @@
{
"$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
"extends": ["@datamine/config/biome"]
}

34
apps/bot/package.json Normal file
View file

@ -0,0 +1,34 @@
{
"name": "@datamine/bot",
"packageManager": "yarn@4.1.1",
"private": true,
"type": "module",
"main": "./dist/main.js",
"module": "./dist/main.js",
"types": "./dist/main.d.ts",
"exports": [
"./dist/commands/subscribe.js",
"./dist/commands/unsubscribe.js"
],
"devDependencies": {
"@biomejs/biome": "1.7.0",
"@datamine/config": "workspace:*",
"@types/node": "20.12.7",
"discord-api-types": "0.37.79",
"pkgroll": "2.0.2",
"tsx": "4.7.2",
"typescript": "5.4.5"
},
"scripts": {
"dev": "tsx ./src/main.ts",
"lint": "biome check ./src/**/*",
"build": "pkgroll"
},
"dependencies": {
"@datamine/database": "workspace:*",
"@sapphire/framework": "5.1.0",
"discord.js": "14.x",
"dotenv": "16.4.5",
"drizzle-orm": "0.30.8"
}
}

View file

@ -0,0 +1,83 @@
import { database, Schema } from "@datamine/database";
import { Command } from "@sapphire/framework";
import { ChannelType } from "discord.js";
export class SubscribeCommand extends Command {
public constructor(
context: Command.LoaderContext,
options: Command.Options,
) {
super(context, { ...options });
}
public override registerApplicationCommands(
registry: Command.Registry,
) {
registry.registerChatInputCommand((builder) =>
builder
.setName("subscribe")
.setDescription("Subscribe to the Datamine updates.")
.addChannelOption((option) =>
option
.setName("channel")
.setDescription("The channel to send updates to.")
.addChannelTypes(ChannelType.GuildText),
)
.addRoleOption((option) =>
option
.setName("role")
.setDescription("The role to send updates to."),
),
);
}
public async chatInputRun(
interaction: Command.ChatInputCommandInteraction,
) {
if (!interaction.inGuild()) return;
if (!interaction.memberPermissions.has("ManageWebhooks")) {
return interaction.reply({
content: "You do not have permission to use this command.",
ephemeral: true,
});
}
const channel = interaction.options.getChannel("channel", false, [
ChannelType.GuildText,
]);
const role = interaction.options.getRole("role");
const data = {
channel: channel
? BigInt(channel.id)
: BigInt(interaction.channelId),
role: role ? BigInt(role.id) : undefined,
};
try {
await database
.insert(Schema.servers)
.values({
id: BigInt(interaction.guildId),
...data,
})
.onConflictDoUpdate({
target: Schema.servers.id,
set: data,
});
} catch (error) {
return interaction.reply({
content:
"An error occurred while subscribing to Datamine updates.",
ephemeral: true,
});
}
return interaction.reply({
content: `Datamine posts will now be posted into <#${
data.channel
}> ${data.role ? `and mention <@&${data.role}>` : ""}.`,
ephemeral: true,
});
}
}

View file

@ -0,0 +1,53 @@
import { Drizzle, Schema, database } from "@datamine/database";
import { Command } from "@sapphire/framework";
export class SubscribeCommand extends Command {
public constructor(
context: Command.LoaderContext,
options: Command.Options,
) {
super(context, { ...options });
}
public override registerApplicationCommands(
registry: Command.Registry,
) {
registry.registerChatInputCommand((builder) =>
builder
.setName("unsubscribe")
.setDescription("Unsubscribe from the Datamine updates."),
);
}
public async chatInputRun(
interaction: Command.ChatInputCommandInteraction,
) {
if (!interaction.inGuild()) return;
if (!interaction.memberPermissions.has("ManageWebhooks")) {
return interaction.reply({
content: "You do not have permission to use this command.",
ephemeral: true,
});
}
try {
await database
.delete(Schema.servers)
.where(
Drizzle.eq(Schema.servers.id, BigInt(interaction.guildId)),
);
} catch (error) {
return interaction.reply({
content:
"An error occurred while subscribing to Datamine updates.",
ephemeral: true,
});
}
return interaction.reply({
content:
"Datamine posts will no longer be posted in this server.",
ephemeral: true,
});
}
}

15
apps/bot/src/main.ts Normal file
View file

@ -0,0 +1,15 @@
import "dotenv/config";
import { SapphireClient } from "@sapphire/framework";
import { resolve } from "node:path";
if (!process.env.DISCORD_BOT_TOKEN) {
throw new Error("DISCORD_BOT_TOKEN is not defined");
}
const client = new SapphireClient({
intents: [],
baseUserDirectory: resolve(import.meta.dirname),
});
client.login(process.env.DISCORD_BOT_TOKEN);

5
apps/bot/tsconfig.json Normal file
View file

@ -0,0 +1,5 @@
{
"extends": "@datamine/config/typescript",
"exclude": ["node_modules"],
"include": ["src"]
}

38
apps/dispatch/Dockerfile Normal file
View file

@ -0,0 +1,38 @@
FROM node:21-alpine AS base
FROM base AS builder
RUN apk add --no-cache libc6-compat
RUN apk update
RUN yarn set version canary
RUN yarn config set nodeLinker node-modules
WORKDIR /app
COPY . .
RUN yarn dlx turbo prune @datamine/dispatch --docker
FROM base AS installer
RUN apk add --no-cache libc6-compat
RUN apk update
WORKDIR /app
COPY .gitignore .gitignore
COPY --from=builder /app/out/json/ .
COPY --from=builder /app/out/yarn.lock ./yarn.lock
RUN yarn install
COPY --from=builder /app/out/full/ .
COPY turbo.json turbo.json
RUN yarn turbo build --filter=dispatch...
RUN yarn workspace @datamine/database run drizzle
FROM base AS runner
WORKDIR /app
# Don't run production as root
RUN addgroup --system --gid 1001 datamine
RUN adduser --system --uid 1001 datamine
USER datamine
COPY --from=installer /app .
CMD node apps/dispatch/dist/main.js

1
apps/dispatch/README.md Normal file
View file

@ -0,0 +1 @@
# @datamine/dispatch

4
apps/dispatch/biome.json Normal file
View file

@ -0,0 +1,4 @@
{
"$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
"extends": ["@datamine/config/biome"]
}

View file

@ -0,0 +1,36 @@
{
"name": "@datamine/dispatch",
"packageManager": "yarn@4.1.1",
"private": true,
"type": "module",
"main": "./dist/main.js",
"module": "./dist/main.js",
"types": "./dist/main.d.ts",
"exports": {
"import": {
"types": "./dist/main.d.ts"
}
},
"devDependencies": {
"@biomejs/biome": "1.7.0",
"@datamine/config": "workspace:*",
"@types/node": "20.12.7",
"discord-api-types": "0.37.79",
"pkgroll": "2.0.2",
"tsx": "4.7.2",
"typescript": "5.4.5"
},
"scripts": {
"dev": "tsx ./src/main.ts",
"lint": "biome check ./src/**/*",
"build": "pkgroll"
},
"dependencies": {
"@datamine/database": "workspace:*",
"@discordjs/rest": "2.2.0",
"@hono/node-server": "1.10.0",
"dotenv": "16.4.5",
"effect": "3.0.0",
"hono": "4.2.4"
}
}

55
apps/dispatch/src/main.ts Normal file
View file

@ -0,0 +1,55 @@
import "dotenv/config";
import { database } from "@datamine/database";
import { REST } from "@discordjs/rest";
import { serve } from "@hono/node-server";
import { Routes, type APIEmbed } from "discord-api-types/v10";
import { Hono } from "hono";
import { validator } from "hono/validator";
if (!process.env.DISCORD_BOT_TOKEN) {
throw new Error("DISCORD_BOT_TOKEN is not defined");
}
const rest = new REST({ version: "10" }).setToken(
// biome-ignore lint/style/noNonNullAssertion: check is ran before the function
process.env.DISCORD_BOT_TOKEN!,
);
const app = new Hono();
const dispatch = app
.post(
"/ingest",
validator<APIEmbed, string, string, "json", Promise<APIEmbed>>(
"json",
async (value) => {
return value;
},
),
async (c) => {
const body = await c.req.valid("json");
const servers = await database.query.servers.findMany();
for (const server of servers) {
rest.post(Routes.channelMessages(`${BigInt(server.channel)}`), {
body: { embeds: [body] },
});
}
return c.text(`Fanning out to ${servers.length} servers.`);
},
)
.post("/announcement", async (c) => {
return c.text("Not Implemented");
});
serve(
{
...app,
port: Number(process.env.PORT ?? 5001),
},
(addr) => {
console.log(`Listening on http://localhost:${addr.port}`);
},
);
export type Dispatch = typeof dispatch;

View file

@ -0,0 +1,5 @@
{
"extends": "@datamine/config/typescript",
"exclude": ["node_modules"],
"include": ["src"]
}

37
apps/ingest/Dockerfile Normal file
View file

@ -0,0 +1,37 @@
FROM node:21-alpine AS base
FROM base AS builder
RUN apk add --no-cache libc6-compat
RUN apk update
RUN yarn set version canary
RUN yarn config set nodeLinker node-modules
WORKDIR /app
COPY . .
RUN yarn dlx turbo prune @datamine/ingest --docker
FROM base AS installer
RUN apk add --no-cache libc6-compat
RUN apk update
WORKDIR /app
COPY .gitignore .gitignore
COPY --from=builder /app/out/json/ .
COPY --from=builder /app/out/yarn.lock ./yarn.lock
RUN yarn install
COPY --from=builder /app/out/full/ .
COPY turbo.json turbo.json
RUN yarn turbo build --filter=ingest...
FROM base AS runner
WORKDIR /app
# Don't run production as root
RUN addgroup --system --gid 1001 datamine
RUN adduser --system --uid 1001 datamine
USER datamine
COPY --from=installer /app .
CMD node apps/ingest/dist/main.js

1
apps/ingest/README.md Normal file
View file

@ -0,0 +1 @@
# @datamine/ingest

4
apps/ingest/biome.json Normal file
View file

@ -0,0 +1,4 @@
{
"$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
"extends": ["@datamine/config/biome"]
}

36
apps/ingest/package.json Normal file
View file

@ -0,0 +1,36 @@
{
"name": "@datamine/ingest",
"packageManager": "yarn@4.1.1",
"private": true,
"type": "module",
"main": "./dist/main.js",
"module": "./dist/main.js",
"types": "./dist/main.d.ts",
"exports": {
"import": {
"types": "./dist/main.d.ts"
}
},
"devDependencies": {
"@biomejs/biome": "1.6.4",
"@datamine/config": "workspace:*",
"@octokit/webhooks-types": "7.5.0",
"@types/node": "20.12.7",
"discord-api-types": "0.37.79",
"pkgroll": "2.0.2",
"tsx": "4.7.2",
"typescript": "5.4.5"
},
"scripts": {
"dev": "tsx ./src/main.ts",
"lint": "biome check ./src/**/*",
"build": "pkgroll"
},
"dependencies": {
"@datamine/database": "workspace:*",
"@datamine/dispatch": "workspace:*",
"@hono/node-server": "1.10.0",
"dotenv": "16.4.5",
"hono": "4.2.4"
}
}

75
apps/ingest/src/main.ts Normal file
View file

@ -0,0 +1,75 @@
import "dotenv/config";
import type { Dispatch } from "@datamine/dispatch";
import { serve } from "@hono/node-server";
import type { CommitCommentEvent } from "@octokit/webhooks-types";
import { Hono } from "hono";
import { hc } from "hono/client";
import { verifyGithub } from "./util/verifyGithub.ts";
const dispatch = hc<Dispatch>(
process.env.DISPATCH_URL ?? "http://dispatch:5001/",
);
const app = new Hono();
const ingest = app.post("/", async (c) => {
const event: CommitCommentEvent = await c.req.json();
const verified = verifyGithub(
c.req.header("x-hub-signature-256") as string,
event,
);
if (!verified) {
return c.json({ error: "Invalid signature" }, 401);
}
if (c.req.header("x-github-event") !== "commit_comment") {
return c.json({ error: "Invalid event" }, 400);
}
if (
event.repository.name !==
(process.env.GITHUB_REPOSITORY_NAME ?? "Discord-Datamining")
) {
return c.json({ error: "Invalid repository" }, 401);
}
const res = await dispatch.ingest.$post({
json: {
title: `[${event.repository.owner.login}/${
event.repository.name
}] New comment on commit \`${event.comment.commit_id.substring(
0,
7,
)}\``,
description:
event.comment.body.length > 4091
? `${event.comment.body.substring(0, 4091)}\n\`\`\``
: event.comment.body,
url: event.comment.html_url,
author: {
name: event.comment.user.login,
icon_url: event.comment.user.avatar_url,
url: event.comment.user.html_url,
},
timestamp: new Date(event.comment.created_at).toISOString(),
},
});
if (!res.ok) {
return c.json({ error: "Dispatch failed" }, 500);
}
const data = await res.text();
return c.text(data);
});
serve(
{
...app,
port: Number(process.env.PORT ?? 5000),
},
(addr) => {
console.log(`Listening on http://localhost:${addr.port}`);
},
);
export type Ingest = typeof ingest;

View file

@ -0,0 +1,19 @@
import { createHmac, timingSafeEqual } from "node:crypto";
if (!process.env.GITHUB_WEBHOOK_SECRET) {
throw new Error("GITHUB_WEBHOOK_SECRET is not defined");
}
export function verifyGithub(signature: string, body: object): boolean {
const verify = createHmac(
"sha256",
// biome-ignore lint/style/noNonNullAssertion: check is ran before the function
process.env.GITHUB_WEBHOOK_SECRET!,
)
.update(JSON.stringify(body))
.digest("hex");
return timingSafeEqual(
Buffer.from(signature),
Buffer.from(`sha256=${verify}`),
);
}

View file

@ -0,0 +1,5 @@
{
"extends": "@datamine/config/typescript",
"exclude": ["node_modules"],
"include": ["src"]
}

76
docker-compose.yml Normal file
View file

@ -0,0 +1,76 @@
version: '3.7'
services:
postgres:
container_name: postgres
image: postgres:16-alpine
environment:
- POSTGRES_USER=${POSTGRES_USER:-datamine}
- POSTGRES_DB=${POSTGRES_DB:-datamine}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
restart: always
ports:
- 54320:5432
networks:
- datamine
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 5s
retries: 60
ingest:
container_name: ingest
build:
context: .
dockerfile: ./apps/ingest/Dockerfile
restart: always
ports:
- 5000:5000
environment:
- GITHUB_WEBHOOK_SECRET=${GITHUB_WEBHOOK_SECRET}
- GITHUB_REPOSITORY_NAME=${GITHUB_REPOSITORY_NAME}
- DISPATCH_URL=${DISPATCH_URL:-http://dispatch:5001/}
networks:
- datamine
depends_on:
- dispatch
dispatch:
container_name: dispatch
build:
context: .
dockerfile: ./apps/dispatch/Dockerfile
restart: always
environment:
- DB_HOST=postgres
- DB_PORT=5432
- DB_NAME=${POSTGRES_DB:-datamine}
- DB_USER=${POSTGRES_USER:-datamine}
- DB_PASSWORD=${POSTGRES_PASSWORD}
- DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN}
depends_on:
postgres:
condition: service_healthy
networks:
- datamine
bot:
container_name: bot
build:
context: .
dockerfile: ./apps/bot/Dockerfile
restart: always
environment:
- DB_HOST=postgres
- DB_PORT=5432
- DB_NAME=${POSTGRES_DB:-datamine}
- DB_USER=${POSTGRES_USER:-datamine}
- DB_PASSWORD=${POSTGRES_PASSWORD}
- DISCORD_BOT_TOKEN=${DISCORD_BOT_TOKEN}
depends_on:
postgres:
condition: service_healthy
networks:
- datamine
networks:
datamine:
name: datamine

View file

@ -1,14 +1,26 @@
{
"name": "datamine",
"packageManager": "yarn@4.1.1",
"engines": {
"node": ">=18"
},
"private": true,
"workspaces": ["packages/*"],
"devDependencies": {
"@biomejs/biome": "1.6.4",
"@datamine/config": "workspace:^",
"turbo": "^1.13.2"
}
}
{
"name": "datamine",
"packageManager": "yarn@4.1.1",
"engines": {
"node": ">=18"
},
"type": "module",
"private": true,
"workspaces": [
"packages/*",
"apps/*"
],
"devDependencies": {
"@biomejs/biome": "1.6.4",
"@datamine/config": "workspace:^",
"turbo": "^1.13.2"
},
"scripts": {
"dev": "turbo run dev",
"build": "turbo run build",
"lint": "turbo run lint"
},
"volta": {
"node": "20.12.2"
}
}

View file

@ -1,7 +1,7 @@
{
"$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
"organizeImports": {
"enabled": true
"enabled": false
},
"linter": {
"enabled": true,
@ -18,7 +18,7 @@
},
"formatter": {
"enabled": true,
"lineWidth": 256,
"lineWidth": 72,
"indentStyle": "space",
"formatWithErrors": true,
"indentWidth": 2

View file

@ -1,16 +1,17 @@
{
"compilerOptions": {
"allowImportingTsExtensions": true,
"allowSyntheticDefaultImports": true,
"alwaysStrict": true,
"declaration": true,
"declarationMap": true,
"esModuleInterop": true,
"importHelpers": false,
"incremental": true,
"lib": ["esnext"],
"module": "Node16",
"moduleResolution": "Node16",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"newLine": "lf",
"noEmit": true,
"noEmitHelpers": false,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": true,
@ -22,7 +23,7 @@
"resolveJsonModule": true,
"sourceMap": true,
"strict": true,
"target": "ES2020",
"target": "ESNext",
"useDefineForClassFields": true
}
}

View file

@ -0,0 +1,4 @@
{
"$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
"extends": ["@datamine/config/biome"]
}

View file

@ -0,0 +1,6 @@
import type { Config } from "drizzle-kit";
export default {
schema: "./src/schema.ts",
out: "./migrations",
} satisfies Config;

View file

@ -0,0 +1,34 @@
{
"name": "@datamine/database",
"packageManager": "yarn@4.1.1",
"private": true,
"type": "module",
"main": "./dist/main.js",
"module": "./dist/main.js",
"types": "./dist/main.d.ts",
"exports": {
"import": {
"types": "./dist/main.d.ts",
"default": "./dist/main.js"
}
},
"devDependencies": {
"@biomejs/biome": "1.6.4",
"@datamine/config": "workspace:*",
"@types/node": "20.12.7",
"pkgroll": "2.0.2",
"tsx": "4.7.2",
"typescript": "5.4.5"
},
"scripts": {
"lint": "biome check ./src/**/*",
"build": "pkgroll",
"drizzle": "drizzle-kit generate:pg"
},
"dependencies": {
"@types/pg": "8.11.5",
"drizzle-kit": "0.20.14",
"drizzle-orm": "0.30.8",
"pg": "8.11.5"
}
}

View file

@ -0,0 +1,23 @@
import { drizzle } from "drizzle-orm/node-postgres";
import { migrate } from "drizzle-orm/node-postgres/migrator";
import { resolve } from "node:path";
import pg from "pg";
import * as schema from "./schema.ts";
const client = new pg.Client({
host: process.env.DB_HOST,
port: Number(process.env.DB_PORT),
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
await client.connect();
export const database = drizzle(client, { schema });
await migrate(database, {
migrationsFolder: resolve(import.meta.dirname, "../migrations"),
});
export * as Drizzle from "drizzle-orm";
export * as Schema from "./schema.ts";

View file

@ -0,0 +1,10 @@
import { bigint, pgTable } from "drizzle-orm/pg-core";
export const servers = pgTable("servers", {
id: bigint("id", { mode: "bigint" }).primaryKey(),
channel: bigint("channel", { mode: "bigint" }).notNull(),
role: bigint("role", { mode: "bigint" }),
});
export type Server = typeof servers.$inferSelect;
export type NewServer = typeof servers.$inferInsert;

View file

@ -0,0 +1,5 @@
{
"extends": "@datamine/config/typescript",
"exclude": ["node_modules"],
"include": ["src"]
}

3389
yarn.lock

File diff suppressed because it is too large Load diff