i hyperfocused and here is v1

apps/ingest/Dockerfile

@@ -0,0 +1,37 @@
+FROM node:21-alpine AS base
+
+FROM base AS builder
+RUN apk add --no-cache libc6-compat
+RUN apk update
+RUN yarn set version canary
+RUN yarn config set nodeLinker node-modules
+
+WORKDIR /app
+COPY . .
+RUN yarn dlx turbo prune @datamine/ingest --docker
+
+FROM base AS installer
+RUN apk add --no-cache libc6-compat
+RUN apk update
+WORKDIR /app
+
+COPY .gitignore .gitignore
+COPY --from=builder /app/out/json/ .
+COPY --from=builder /app/out/yarn.lock ./yarn.lock
+RUN yarn install
+
+COPY --from=builder /app/out/full/ .
+COPY turbo.json turbo.json
+
+RUN yarn turbo build --filter=ingest...
+
+FROM base AS runner
+WORKDIR /app
+
+# Don't run production as root
+RUN addgroup --system --gid 1001 datamine
+RUN adduser --system --uid 1001 datamine
+USER datamine
+COPY --from=installer /app .
+
+CMD node apps/ingest/dist/main.js

apps/ingest/README.md

@@ -0,0 +1 @@
+# @datamine/ingest

apps/ingest/biome.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.6.4/schema.json",
+  "extends": ["@datamine/config/biome"]
+}

apps/ingest/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@datamine/ingest",
+  "packageManager": "yarn@4.1.1",
+  "private": true,
+  "type": "module",
+  "main": "./dist/main.js",
+  "module": "./dist/main.js",
+  "types": "./dist/main.d.ts",
+  "exports": {
+    "import": {
+      "types": "./dist/main.d.ts"
+    }
+  },
+  "devDependencies": {
+    "@biomejs/biome": "1.6.4",
+    "@datamine/config": "workspace:*",
+    "@octokit/webhooks-types": "7.5.0",
+    "@types/node": "20.12.7",
+    "discord-api-types": "0.37.79",
+    "pkgroll": "2.0.2",
+    "tsx": "4.7.2",
+    "typescript": "5.4.5"
+  },
+  "scripts": {
+    "dev": "tsx ./src/main.ts",
+    "lint": "biome check ./src/**/*",
+    "build": "pkgroll"
+  },
+  "dependencies": {
+    "@datamine/database": "workspace:*",
+    "@datamine/dispatch": "workspace:*",
+    "@hono/node-server": "1.10.0",
+    "dotenv": "16.4.5",
+    "hono": "4.2.4"
+  }
+}

apps/ingest/src/main.ts

@@ -0,0 +1,75 @@
+import "dotenv/config";
+
+import type { Dispatch } from "@datamine/dispatch";
+import { serve } from "@hono/node-server";
+import type { CommitCommentEvent } from "@octokit/webhooks-types";
+import { Hono } from "hono";
+import { hc } from "hono/client";
+import { verifyGithub } from "./util/verifyGithub.ts";
+
+const dispatch = hc<Dispatch>(
+  process.env.DISPATCH_URL ?? "http://dispatch:5001/",
+);
+const app = new Hono();
+
+const ingest = app.post("/", async (c) => {
+  const event: CommitCommentEvent = await c.req.json();
+  const verified = verifyGithub(
+    c.req.header("x-hub-signature-256") as string,
+    event,
+  );
+
+  if (!verified) {
+    return c.json({ error: "Invalid signature" }, 401);
+  }
+
+  if (c.req.header("x-github-event") !== "commit_comment") {
+    return c.json({ error: "Invalid event" }, 400);
+  }
+
+  if (
+    event.repository.name !==
+    (process.env.GITHUB_REPOSITORY_NAME ?? "Discord-Datamining")
+  ) {
+    return c.json({ error: "Invalid repository" }, 401);
+  }
+
+  const res = await dispatch.ingest.$post({
+    json: {
+      title: `[${event.repository.owner.login}/${
+        event.repository.name
+      }] New comment on commit \`${event.comment.commit_id.substring(
+        0,
+        7,
+      )}\``,
+      description:
+        event.comment.body.length > 4091
+          ? `${event.comment.body.substring(0, 4091)}…\n\`\`\``
+          : event.comment.body,
+      url: event.comment.html_url,
+      author: {
+        name: event.comment.user.login,
+        icon_url: event.comment.user.avatar_url,
+        url: event.comment.user.html_url,
+      },
+      timestamp: new Date(event.comment.created_at).toISOString(),
+    },
+  });
+  if (!res.ok) {
+    return c.json({ error: "Dispatch failed" }, 500);
+  }
+  const data = await res.text();
+  return c.text(data);
+});
+
+serve(
+  {
+    ...app,
+    port: Number(process.env.PORT ?? 5000),
+  },
+  (addr) => {
+    console.log(`Listening on http://localhost:${addr.port}`);
+  },
+);
+
+export type Ingest = typeof ingest;

apps/ingest/src/util/verifyGithub.ts

@@ -0,0 +1,19 @@
+import { createHmac, timingSafeEqual } from "node:crypto";
+
+if (!process.env.GITHUB_WEBHOOK_SECRET) {
+  throw new Error("GITHUB_WEBHOOK_SECRET is not defined");
+}
+
+export function verifyGithub(signature: string, body: object): boolean {
+  const verify = createHmac(
+    "sha256",
+    // biome-ignore lint/style/noNonNullAssertion: check is ran before the function
+    process.env.GITHUB_WEBHOOK_SECRET!,
+  )
+    .update(JSON.stringify(body))
+    .digest("hex");
+  return timingSafeEqual(
+    Buffer.from(signature),
+    Buffer.from(`sha256=${verify}`),
+  );
+}

apps/ingest/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "@datamine/config/typescript",
+  "exclude": ["node_modules"],
+  "include": ["src"]
+}