NuklearRabbitandClaude Opus 5
cb9bdcd713
Every ssh.exec opened its own connection: a TCP handshake, a key exchange and an
authentication round trip per command. A key rotation paid for that eight times,
a deployment six, and refreshing M profile states M times.
Connections are now kept per server. The three risks that made this worth doing
carefully are handled explicitly:
- Staleness. A pooled connection can be dead exactly when it matters. Liveness is
tracked through error, close and end, and a lease that finds a dead entry opens
a new one. The remaining race, where the connection dies between the check and
the command, is caught by the retry rule below.
- Retrying. Only a failure that proves the command never reached the server is
retried, and only once, and only on a connection that was already established
before this call. execClient marks exactly that case, when the channel fails to
open. A command that opened a stream is never repeated, because the server may
already be acting on it - repeating a deployment is not this layer's decision.
Two tests hold that line: widening the rule to any failure fails both.
- Lifetime. Idle connections close after a minute, the pool is reference counted
so a shared connection survives until its last user is done, closeAll runs
during quit, and every pooled client keeps a standing error listener so an
error while idle cannot reach the uncaughtException handler.
A trust-on-first-use connection is never pooled: it was established without
verifying the fingerprint, so it must not serve a later verified call. A change
to host, port, user, auth type, key path or trusted fingerprint invalidates the
pooled connection.
ssh-service coverage rises from 61% to 90% of lines and 97% of functions.
Also in this commit, the smaller items from the same review:
- Diagnostics batched records that queue up while a write is in flight into one
append, and chmod runs once per file instead of once per record. At the debug
level every IPC call writes a line, which is exactly when troubleshooting.
- The set that suppresses duplicate deployment notifications is trimmed instead
of growing for the lifetime of the process.
- The updater kept the same once('error') pattern on its spawned helper that
took the app down through the SSH client.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
373 lines
16 KiB
JavaScript
373 lines
16 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('node:fs/promises');
|
|
const path = require('node:path');
|
|
const os = require('node:os');
|
|
const crypto = require('node:crypto');
|
|
const { createZip } = require('../shared/zip-writer.cjs');
|
|
const { sanitizeForDiagnostics, redactSecrets } = require('./log-redaction.cjs');
|
|
|
|
const LEVELS = { debug: 10, info: 20, warning: 30, error: 40 };
|
|
|
|
function dateKey(value = new Date()) {
|
|
return value.toISOString().slice(0, 10);
|
|
}
|
|
|
|
function byteSizeLabel(bytes) {
|
|
if (bytes < 1024) return `${bytes} B`;
|
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
|
|
}
|
|
|
|
function safeJson(value) {
|
|
return `${JSON.stringify(value, null, 2)}\n`;
|
|
}
|
|
|
|
function auditBundleEntries(entries, secrets = []) {
|
|
const candidates = [...new Set((secrets || []).map((item) => String(item || '').trim()).filter((item) => item.length >= 4))];
|
|
const findings = [];
|
|
for (const entry of entries) {
|
|
const text = Buffer.isBuffer(entry.data) ? entry.data.toString('utf8') : String(entry.data ?? '');
|
|
for (const secret of candidates) {
|
|
if (text.includes(secret)) findings.push({ file: entry.name, type: 'known-runtime-secret' });
|
|
}
|
|
if (/-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/i.test(text)) findings.push({ file: entry.name, type: 'private-key-marker' });
|
|
if (/https?:\/\/[^\s:@/]+:(?!\[REDACTED\])[^@\s/]+@/i.test(text)) findings.push({ file: entry.name, type: 'url-credential' });
|
|
}
|
|
return { passed: findings.length === 0, checkedFiles: entries.length, knownRuntimeSecretCount: candidates.length, findings };
|
|
}
|
|
|
|
class DiagnosticsService {
|
|
constructor({ userDataPath, appInfo = {}, secretProvider = () => [], preferencesProvider = () => ({}) }) {
|
|
this.userDataPath = userDataPath;
|
|
this.logDirectory = path.join(userDataPath, 'diagnostics');
|
|
this.appInfo = appInfo;
|
|
this.secretProvider = secretProvider;
|
|
this.preferencesProvider = preferencesProvider;
|
|
this.sessionId = crypto.randomUUID();
|
|
this.writeChain = Promise.resolve();
|
|
this.pendingLines = [];
|
|
this.pendingFlush = null;
|
|
this.securedFiles = new Set();
|
|
this.initialized = false;
|
|
this.lastWriteError = null;
|
|
this.lastBundlePath = null;
|
|
}
|
|
|
|
preferences() {
|
|
const source = this.preferencesProvider?.() || {};
|
|
return {
|
|
enabled: source.diagnosticsEnabled !== false,
|
|
level: ['debug', 'info', 'warning', 'error'].includes(source.diagnosticLevel) ? source.diagnosticLevel : 'info',
|
|
retentionDays: Math.min(Math.max(Number(source.logRetentionDays) || 14, 1), 90),
|
|
maxFileMb: Math.min(Math.max(Number(source.maxLogFileMb) || 8, 1), 50)
|
|
};
|
|
}
|
|
|
|
sanitize(value, options = {}) {
|
|
return sanitizeForDiagnostics(value, {
|
|
secrets: this.secretProvider?.() || [],
|
|
homeDir: os.homedir(),
|
|
cwd: process.cwd(),
|
|
...options
|
|
});
|
|
}
|
|
|
|
async initialize() {
|
|
await fs.mkdir(this.logDirectory, { recursive: true, mode: 0o700 });
|
|
try { await fs.chmod(this.logDirectory, 0o700); } catch {}
|
|
this.initialized = true;
|
|
await this.prune();
|
|
await this.info('diagnostics.session.started', {
|
|
sessionId: this.sessionId,
|
|
app: this.appInfo,
|
|
platform: process.platform,
|
|
arch: process.arch,
|
|
node: process.versions.node,
|
|
electron: process.versions.electron || null
|
|
});
|
|
}
|
|
|
|
shouldWrite(level) {
|
|
const preferences = this.preferences();
|
|
return preferences.enabled && LEVELS[level] >= LEVELS[preferences.level];
|
|
}
|
|
|
|
filePathForToday() {
|
|
return path.join(this.logDirectory, `forgeflow-${dateKey()}.jsonl`);
|
|
}
|
|
|
|
async rotateIfNeeded(filePath) {
|
|
const limit = this.preferences().maxFileMb * 1024 * 1024;
|
|
const stat = await fs.stat(filePath).catch(() => null);
|
|
if (!stat || stat.size < limit) return filePath;
|
|
for (let index = 1; index < 100; index += 1) {
|
|
const candidate = path.join(this.logDirectory, `forgeflow-${dateKey()}-${String(index).padStart(2, '0')}.jsonl`);
|
|
const candidateStat = await fs.stat(candidate).catch(() => null);
|
|
if (!candidateStat || candidateStat.size < limit) return candidate;
|
|
}
|
|
return path.join(this.logDirectory, `forgeflow-${dateKey()}-${Date.now()}.jsonl`);
|
|
}
|
|
|
|
log(level, event, details = {}) {
|
|
if (!this.shouldWrite(level)) return Promise.resolve(false);
|
|
const record = this.sanitize({
|
|
timestamp: new Date().toISOString(),
|
|
level,
|
|
event: String(event || 'diagnostics.event').slice(0, 160),
|
|
sessionId: this.sessionId,
|
|
details
|
|
});
|
|
this.pendingLines.push(`${JSON.stringify(record)}\n`);
|
|
// At the debug level every IPC call and every Gitea request writes a line.
|
|
// Records that queue up while a write is in flight are appended together, so
|
|
// a burst costs one open/write/close instead of one per record.
|
|
if (this.pendingFlush) return this.pendingFlush;
|
|
this.pendingFlush = this.writeChain.then(async () => {
|
|
this.pendingFlush = null;
|
|
const lines = this.pendingLines.splice(0).join('');
|
|
if (!lines) return true;
|
|
try {
|
|
if (!this.initialized) await fs.mkdir(this.logDirectory, { recursive: true, mode: 0o700 });
|
|
const target = await this.rotateIfNeeded(this.filePathForToday());
|
|
await fs.appendFile(target, lines, { encoding: 'utf8', mode: 0o600 });
|
|
// The mode above only applies when appendFile creates the file, so the
|
|
// explicit chmod is needed once per file rather than once per record.
|
|
if (!this.securedFiles.has(target)) {
|
|
try { await fs.chmod(target, 0o600); } catch { /* best effort */ }
|
|
this.securedFiles.add(target);
|
|
}
|
|
this.lastWriteError = null;
|
|
return true;
|
|
} catch (error) {
|
|
this.lastWriteError = error.message;
|
|
return false;
|
|
}
|
|
});
|
|
this.writeChain = this.pendingFlush.catch(() => {});
|
|
return this.pendingFlush;
|
|
}
|
|
|
|
debug(event, details) { return this.log('debug', event, details); }
|
|
info(event, details) { return this.log('info', event, details); }
|
|
warning(event, details) { return this.log('warning', event, details); }
|
|
error(event, details) { return this.log('error', event, details); }
|
|
|
|
async flush() {
|
|
await this.writeChain;
|
|
}
|
|
|
|
async listLogFiles() {
|
|
await fs.mkdir(this.logDirectory, { recursive: true, mode: 0o700 });
|
|
const entries = await fs.readdir(this.logDirectory, { withFileTypes: true });
|
|
const files = [];
|
|
for (const entry of entries) {
|
|
if (!entry.isFile() || !/^forgeflow-.*\.jsonl$/i.test(entry.name)) continue;
|
|
const absolute = path.join(this.logDirectory, entry.name);
|
|
const stat = await fs.stat(absolute).catch(() => null);
|
|
if (stat) files.push({ name: entry.name, path: absolute, size: stat.size, modifiedAt: stat.mtime.toISOString() });
|
|
}
|
|
return files.sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt));
|
|
}
|
|
|
|
async prune() {
|
|
const cutoff = Date.now() - this.preferences().retentionDays * 24 * 60 * 60 * 1000;
|
|
for (const file of await this.listLogFiles()) {
|
|
if (new Date(file.modifiedAt).getTime() < cutoff) await fs.rm(file.path, { force: true }).catch(() => {});
|
|
}
|
|
}
|
|
|
|
async getStatus() {
|
|
await this.flush();
|
|
const files = await this.listLogFiles();
|
|
const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
|
|
return {
|
|
enabled: this.preferences().enabled,
|
|
level: this.preferences().level,
|
|
retentionDays: this.preferences().retentionDays,
|
|
maxFileMb: this.preferences().maxFileMb,
|
|
directory: this.sanitize(this.logDirectory),
|
|
fileCount: files.length,
|
|
totalBytes,
|
|
totalSize: byteSizeLabel(totalBytes),
|
|
latestAt: files[0]?.modifiedAt || null,
|
|
lastWriteError: this.lastWriteError
|
|
};
|
|
}
|
|
|
|
async clear() {
|
|
await this.flush();
|
|
for (const file of await this.listLogFiles()) await fs.rm(file.path, { force: true });
|
|
await this.info('diagnostics.logs.cleared', {});
|
|
return this.getStatus();
|
|
}
|
|
|
|
async collectLogs(maxBytes = 20 * 1024 * 1024) {
|
|
await this.flush();
|
|
const output = [];
|
|
let used = 0;
|
|
for (const file of await this.listLogFiles()) {
|
|
if (used >= maxBytes) break;
|
|
const remaining = maxBytes - used;
|
|
const content = await fs.readFile(file.path);
|
|
const slice = content.length > remaining ? content.subarray(content.length - remaining) : content;
|
|
output.push({ name: `logs/${file.name}`, data: Buffer.from(redactSecrets(slice.toString('utf8'), this.secretProvider?.() || []), 'utf8') });
|
|
used += slice.length;
|
|
}
|
|
return output;
|
|
}
|
|
|
|
async exportSupportBundle({ destinationPath, publicState, repositories = [], operations = [], preflight = null, privacyMode = 'standard', extra = {} }) {
|
|
if (!destinationPath) throw new Error('No support bundle destination was selected.');
|
|
if (!['standard', 'strict'].includes(privacyMode)) throw new Error('Unsupported diagnostic privacy mode.');
|
|
if (path.extname(destinationPath).toLowerCase() !== '.zip') throw new Error('Diagnostic bundles must use the .zip extension.');
|
|
await this.info('diagnostics.bundle.requested', { privacyMode, repositoryCount: repositories.length, operationCount: operations.length });
|
|
const strict = privacyMode === 'strict';
|
|
const sanitize = (value) => this.sanitize(value, { strictIdentifiers: strict });
|
|
const generatedAt = new Date().toISOString();
|
|
const diagnosticsStatus = await this.getStatus();
|
|
const system = sanitize({
|
|
app: this.appInfo,
|
|
generatedAt,
|
|
sessionId: this.sessionId,
|
|
platform: process.platform,
|
|
arch: process.arch,
|
|
release: os.release(),
|
|
type: os.type(),
|
|
cpus: os.cpus()?.map((cpu) => cpu.model).filter((value, index, array) => array.indexOf(value) === index),
|
|
cpuCount: os.cpus()?.length || null,
|
|
totalMemoryBytes: os.totalmem(),
|
|
freeMemoryBytes: os.freemem(),
|
|
uptimeSeconds: os.uptime(),
|
|
locale: Intl.DateTimeFormat().resolvedOptions().locale,
|
|
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
versions: process.versions
|
|
});
|
|
|
|
const sanitizedState = sanitize(publicState || {});
|
|
if (sanitizedState.gitea) sanitizedState.gitea.hasToken = Boolean(publicState?.gitea?.hasToken);
|
|
const sanitizedRepositories = sanitize(repositories.map((repository) => ({
|
|
id: repository.id,
|
|
fullName: repository.fullName,
|
|
linkState: repository.linkState,
|
|
localPath: repository.localPath,
|
|
attention: repository.attention,
|
|
attentionReason: repository.attentionReason,
|
|
readyToDeploy: repository.readyToDeploy,
|
|
localStatus: repository.localStatus ? {
|
|
branch: repository.localStatus.branch,
|
|
head: repository.localStatus.head,
|
|
counts: repository.localStatus.counts,
|
|
clean: repository.localStatus.clean,
|
|
remoteUrl: repository.localStatus.remoteUrl
|
|
} : null,
|
|
deploymentProfiles: repository.deploymentProfiles?.map((profile) => ({
|
|
id: profile.id,
|
|
name: profile.name,
|
|
environment: profile.environment,
|
|
branch: profile.branch,
|
|
workflowFile: profile.workflowFile,
|
|
rollbackWorkflowFile: profile.rollbackWorkflowFile,
|
|
healthcheckUrl: profile.healthcheckUrl,
|
|
statusUrl: profile.statusUrl,
|
|
state: profile.state
|
|
})) || []
|
|
})));
|
|
const sanitizedOperations = sanitize(operations.map((operation) => ({
|
|
id: operation.id,
|
|
type: operation.type,
|
|
action: operation.action,
|
|
status: operation.status,
|
|
repository: operation.repository,
|
|
profileId: operation.profileId,
|
|
profileName: operation.profileName,
|
|
environment: operation.environment,
|
|
workflowFile: operation.workflowFile,
|
|
branch: operation.branch,
|
|
sha: operation.sha,
|
|
shortSha: operation.shortSha,
|
|
createdAt: operation.createdAt,
|
|
updatedAt: operation.updatedAt,
|
|
dispatchedAt: operation.dispatchedAt,
|
|
stages: operation.stages,
|
|
jobs: operation.jobs,
|
|
logs: operation.logs,
|
|
failure: operation.failure,
|
|
pollError: operation.pollError,
|
|
applicationState: operation.applicationState,
|
|
run: operation.run ? {
|
|
id: operation.run.id,
|
|
runNumber: operation.run.runNumber,
|
|
name: operation.run.name,
|
|
status: operation.run.status,
|
|
conclusion: operation.run.conclusion,
|
|
headSha: operation.run.headSha,
|
|
headBranch: operation.run.headBranch,
|
|
workflowPath: operation.run.workflowPath,
|
|
createdAt: operation.run.createdAt,
|
|
updatedAt: operation.run.updatedAt
|
|
} : null,
|
|
runnerLog: operation.runnerLog ? {
|
|
included: false,
|
|
reason: 'Raw runner output is intentionally omitted from diagnostic bundles.',
|
|
characters: String(operation.runnerLog).length,
|
|
lines: String(operation.runnerLog).split(/\r?\n/).length
|
|
} : null
|
|
})));
|
|
const manifest = {
|
|
schemaVersion: 1,
|
|
product: 'ForgeFlow Support Bundle',
|
|
generatedAt,
|
|
privacyMode,
|
|
containsSecrets: false,
|
|
redaction: {
|
|
knownRuntimeSecrets: true,
|
|
sensitiveObjectKeys: true,
|
|
authorizationHeaders: true,
|
|
credentialUrls: true,
|
|
privateKeys: true,
|
|
userHomePaths: true,
|
|
identifiersHashed: strict
|
|
},
|
|
files: []
|
|
};
|
|
|
|
const entries = [
|
|
{ name: 'README.txt', data: `ForgeFlow diagnostic support bundle\nGenerated: ${generatedAt}\nPrivacy mode: ${privacyMode}\n\nThis bundle is generated locally. Access tokens, passwords, authorization headers, embedded URL credentials, encrypted token blobs and private keys are removed. Review the bundle before sharing it.\n` },
|
|
{ name: 'system.json', data: safeJson(system) },
|
|
{ name: 'diagnostics-status.json', data: safeJson(sanitize(diagnosticsStatus)) },
|
|
{ name: 'configuration-sanitized.json', data: safeJson(sanitizedState) },
|
|
{ name: 'repositories-sanitized.json', data: safeJson(sanitizedRepositories) },
|
|
{ name: 'operations-sanitized.json', data: safeJson(sanitizedOperations) },
|
|
{ name: 'preflight.json', data: safeJson(sanitize(preflight || {})) },
|
|
{ name: 'context.json', data: safeJson(sanitize(extra || {})) },
|
|
...(await this.collectLogs())
|
|
];
|
|
|
|
const safetyAudit = auditBundleEntries(entries, this.secretProvider?.() || []);
|
|
if (!safetyAudit.passed) {
|
|
await this.error('diagnostics.bundle.safety-check-failed', { findings: safetyAudit.findings });
|
|
throw new Error('The diagnostic bundle failed its local secret-safety check and was not written.');
|
|
}
|
|
entries.push({ name: 'safety-audit.json', data: safeJson(safetyAudit) });
|
|
manifest.files = entries.map((entry) => ({ name: entry.name, bytes: Buffer.byteLength(entry.data) }));
|
|
entries.unshift({ name: 'manifest.json', data: safeJson(manifest) });
|
|
const archive = createZip(entries);
|
|
const temporary = `${destinationPath}.${process.pid}.${Date.now()}.tmp`;
|
|
await fs.mkdir(path.dirname(destinationPath), { recursive: true });
|
|
await fs.writeFile(temporary, archive, { mode: 0o600 });
|
|
await fs.rename(temporary, destinationPath);
|
|
try { await fs.chmod(destinationPath, 0o600); } catch {}
|
|
this.lastBundlePath = path.resolve(destinationPath);
|
|
const sha256 = crypto.createHash('sha256').update(archive).digest('hex');
|
|
await this.info('diagnostics.bundle.created', { destinationPath, bytes: archive.length, sha256, privacyMode });
|
|
return { path: destinationPath, bytes: archive.length, size: byteSizeLabel(archive.length), sha256, privacyMode, generatedAt };
|
|
}
|
|
|
|
isKnownBundlePath(filePath) {
|
|
return Boolean(filePath && this.lastBundlePath && path.resolve(filePath) === this.lastBundlePath);
|
|
}
|
|
}
|
|
|
|
module.exports = { DiagnosticsService, dateKey, byteSizeLabel, auditBundleEntries, LEVELS };
|