Initial public ModelForge release

2026-09-01 21:30:16 +02:00
commit 7082ab955a
490 changed files with 104252 additions and 0 deletions
@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { execFileSync } from "node:child_process";
+
+function fail(message) {
+  console.error(message);
+  process.exit(1);
+}
+
+function argument(name) {
+  const index = process.argv.indexOf(name);
+  return index >= 0 ? process.argv[index + 1] : undefined;
+}
+
+const root = fs.realpathSync(argument("--repository") ?? process.cwd());
+const outputArgument = argument("--output");
+const reportArgument = argument("--report");
+const allowlistPath = path.resolve(root, argument("--allowlist") ?? "public-source.allowlist");
+if (!outputArgument) fail("Usage: node scripts/export-public-source.mjs --output <new-directory> [--report <file>]");
+
+const output = path.resolve(outputArgument);
+const reportPath = reportArgument ? path.resolve(reportArgument) : undefined;
+if (output === root || output.startsWith(`${root}${path.sep}`)) {
+  fail("The public export must be outside the canonical repository.");
+}
+if (fs.existsSync(output)) fail(`Output already exists: ${output}`);
+const dirty = execFileSync("git", ["-C", root, "status", "--porcelain"], { encoding: "utf8" });
+if (dirty.trim()) {
+  fail("Public export blocked: commit the exact source tree before exporting it.");
+}
+const licensePath = path.join(root, "LICENSE");
+if (!fs.existsSync(licensePath)) fail("Public export blocked: canonical LICENSE is missing.");
+const licenseDigest = crypto.createHash("sha256").update(fs.readFileSync(licensePath)).digest("hex");
+if (licenseDigest !== "0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0") {
+  fail("Public export blocked: LICENSE is not the approved canonical AGPL-3.0 text.");
+}
+
+const allowlist = fs.readFileSync(allowlistPath, "utf8")
+  .split(/\r?\n/u)
+  .map((line) => line.trim())
+  .filter((line) => line && !line.startsWith("#"));
+
+function selected(relativePath) {
+  return allowlist.some((rule) =>
+    rule.endsWith("/**") ? relativePath.startsWith(rule.slice(0, -2)) : relativePath === rule
+  );
+}
+
+const deniedPrefixes = [
+  ".agents/", ".claude/", ".codex/", "artifacts/", "docs/quality/",
+  "reports/"
+];
+
+// The canonical source contains useful integration examples and operational tests tied to the
+// private deployment. The public root keeps the behavior while replacing those identifiers with
+// stable documentation-only examples. Transform before scanning and before hashing the export.
+const replacements = [
+  [/ssh:\/\/git@192\.168\.10\.150:222\/Jens\/ITWorx-ModelForge\.git/gu, "https://git.example.com/example/modelforge.git"],
+  [/https:\/\/gitea\.itworx\.tech\/Jens\/ITWorx-ModelForge/gu, "https://git.example.com/example/modelforge"],
+  [/git@gitea\.itworx\.tech:Jens\/ExampleRAG\.git/gu, "git@git.example.com:example/example-rag.git"],
+  [/modelforge\.itworx\.tech/giu, "modelforge.example.com"],
+  [/gitea\.itworx\.tech/giu, "git.example.com"],
+  [/192\.168\.10\.150/gu, "192.0.2.10"],
+  [/192\.168\.10\.241/gu, "192.0.2.11"],
+  [/192\.168\.10\.3/gu, "192.0.2.53"],
+  [/\bRAGCORE\b/gu, "EXAMPLE_RAG"],
+  [/\bRAGcore\b/gu, "ExampleRAG"],
+  [/\bragcore\b/gu, "examplerag"],
+  [/\bPOKEVAULT\b/gu, "EXAMPLE_VISION"],
+  [/\bPokeVault\b/gu, "ExampleVision"],
+  [/\bpokevault\b/gu, "examplevision"],
+  [/\bNETOPS-FORGE\b/gu, "EXAMPLE-OPS"],
+  [/\bNetOps Forge\b/gu, "ExampleOps"],
+  [/\bnetops-forge\b/gu, "example-ops"],
+  [/\bTOWER\b/gu, "GPU_NODE"],
+  [/\bTower\b/gu, "GPU Node"],
+  [/\btower\b/gu, "gpu_node"]
+];
+
+const managedValidationWorkflow = ".gitea/workflows/managed-validation.yml";
+const pullRequestTrigger = /\n  pull_request:\r?\n/u;
+
+function publicPath(relativePath) {
+  let value = relativePath;
+  for (const [expression, replacement] of replacements) value = value.replace(expression, replacement);
+  return value;
+}
+
+function publicBytes(relativePath, sourceBytes) {
+  if (sourceBytes.includes(0)) return sourceBytes;
+  let value = sourceBytes.toString("utf8");
+  for (const [expression, replacement] of replacements) value = value.replace(expression, replacement);
+  if (relativePath === managedValidationWorkflow) {
+    if (!pullRequestTrigger.test(value)) {
+      fail(`Public export blocked: ${managedValidationWorkflow} has no expected pull_request trigger.`);
+    }
+    value = value.replace(
+      pullRequestTrigger,
+      "\n  # Public exports require explicit owner dispatch; fork PRs never reach private runners.\n"
+    );
+  }
+  return Buffer.from(value, "utf8");
+}
+
+const privatePatterns = [
+  { id: "private-ipv4", expression: /\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|192\.168(?:\.\d{1,3}){2})\b/u },
+  { id: "private-git-host", expression: /\bgitea\.itworx\.tech\b/iu },
+  { id: "personal-windows-path", expression: /[A-Z]:\\Users\\[^\\\s]+\\/iu },
+  { id: "private-ssh-command", expression: /\bssh\s+(?:root@|unraid\b)/iu },
+  { id: "private-node-name", expression: /\btower\b/iu },
+  { id: "private-integration-name", expression: /\b(?:examplerag|examplevision|example-ops)\b/iu }
+];
+
+const tracked = execFileSync("git", ["-C", root, "ls-files", "-z"], { encoding: "utf8" })
+  .split("\0")
+  .filter(Boolean)
+  .map((entry) => entry.replaceAll("\\", "/"));
+const files = tracked.filter(selected).sort((left, right) => left.localeCompare(right, "en"));
+if (files.length === 0) fail("The allowlist selected no tracked files.");
+
+const findings = [];
+const rendered = new Map();
+for (const relativePath of files) {
+  if (deniedPrefixes.some((prefix) => relativePath.startsWith(prefix))) {
+    findings.push({ rule: "denied-path", path: relativePath });
+    continue;
+  }
+  const source = path.join(root, ...relativePath.split("/"));
+  const stat = fs.lstatSync(source);
+  if (!stat.isFile() || stat.isSymbolicLink()) {
+    findings.push({ rule: "non-regular-file", path: relativePath });
+    continue;
+  }
+  if (stat.size > 10 * 1024 * 1024) {
+    findings.push({ rule: "oversized-file", path: relativePath, bytes: stat.size });
+    continue;
+  }
+  const bytes = publicBytes(relativePath, fs.readFileSync(source));
+  rendered.set(relativePath, bytes);
+  if (!bytes.includes(0)) {
+    const text = bytes.toString("utf8");
+    for (const rule of privatePatterns) {
+      if (rule.expression.test(text)) findings.push({ rule: rule.id, path: relativePath });
+    }
+  }
+}
+
+const destinations = new Map();
+for (const relativePath of files) {
+  const destinationPath = publicPath(relativePath);
+  const normalized = path.posix.normalize(destinationPath);
+  if (
+    destinationPath !== normalized ||
+    path.posix.isAbsolute(destinationPath) ||
+    destinationPath.startsWith("../") ||
+    destinationPath.includes("\\")
+  ) {
+    findings.push({ rule: "unsafe-destination-path", path: relativePath, destinationPath });
+    continue;
+  }
+  const collisionKey = destinationPath.toLocaleLowerCase("en-US");
+  const previous = destinations.get(collisionKey);
+  if (previous) {
+    findings.push({
+      rule: "destination-path-collision",
+      path: relativePath,
+      destinationPath,
+      conflictsWith: previous
+    });
+  } else {
+    destinations.set(collisionKey, relativePath);
+  }
+}
+
+for (const required of ["README.md", "SECURITY.md", "CONTRIBUTING.md", "LICENSE", "docker-compose.yml"]) {
+  if (!files.includes(required)) findings.push({ rule: "missing-required-file", path: required });
+}
+
+const sourceRevision = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+const report = { schemaVersion: 1, sourceRevision, selectedFiles: files.length, findings };
+if (reportPath) {
+  fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+  fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+}
+if (findings.length > 0) {
+  fail(`Public export blocked by ${findings.length} finding(s). See ${reportPath ?? "the scan output"}.`);
+}
+
+fs.mkdirSync(output, { recursive: false });
+const manifest = [];
+for (const relativePath of files) {
+  const bytes = rendered.get(relativePath);
+  const destinationPath = publicPath(relativePath);
+  const destination = path.join(output, ...destinationPath.split("/"));
+  fs.mkdirSync(path.dirname(destination), { recursive: true });
+  fs.writeFileSync(destination, bytes);
+  manifest.push({
+    path: destinationPath,
+    bytes: bytes.length,
+    sha256: crypto.createHash("sha256").update(bytes).digest("hex")
+  });
+}
+
+fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_EXPORT.md"),
+  `# Curated public source export\n\nGenerated from private canonical revision \`${sourceRevision}\`.\n\n` +
+  "This parentless candidate excludes private operational history and uses synthetic example identifiers.\n",
+  "utf8");
+fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_MANIFEST.json"),
+  `${JSON.stringify({ schemaVersion: 1, sourceRevision, files: manifest }, null, 2)}\n`, "utf8");
+console.log(`Exported ${files.length} reviewed files to ${output}`);