513 lines
22 KiB
JavaScript
513 lines
22 KiB
JavaScript
'use strict';
|
|
|
|
const fsp = require('node:fs/promises');
|
|
const crypto = require('node:crypto');
|
|
const path = require('node:path').posix;
|
|
|
|
function loadSshModule() {
|
|
try { return require('ssh2'); }
|
|
catch {
|
|
const error = new Error('The ssh2 dependency is not installed. Run npm install before configuring SSH deployments.');
|
|
error.code = 'SSH2_NOT_INSTALLED';
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function loadSshClient() {
|
|
return loadSshModule().Client;
|
|
}
|
|
|
|
function fingerprintKey(key) {
|
|
const buffer = Buffer.isBuffer(key) ? key : Buffer.from(key);
|
|
return `SHA256:${crypto.createHash('sha256').update(buffer).digest('base64').replace(/=+$/, '')}`;
|
|
}
|
|
|
|
function shellQuote(value) {
|
|
return `'${String(value ?? '').replace(/'/g, `'\\''`)}'`;
|
|
}
|
|
|
|
function parseCapabilityOutput(output) {
|
|
const marker = '__FORGEFLOW_SERVER_TEST__';
|
|
const index = String(output || '').lastIndexOf(marker);
|
|
if (index < 0) return { platform: String(output || '').trim(), docker: false, dockerReady: false, compose: false, git: false, tar: false, checksum: false };
|
|
const fields = {};
|
|
for (const line of String(output).slice(index + marker.length).trim().split(/\r?\n/)) {
|
|
const separator = line.indexOf('=');
|
|
if (separator > 0) fields[line.slice(0, separator)] = line.slice(separator + 1);
|
|
}
|
|
const decode = (value) => {
|
|
try { return value ? Buffer.from(value, 'base64').toString('utf8') : ''; }
|
|
catch { return ''; }
|
|
};
|
|
return {
|
|
platform: decode(fields.platform),
|
|
docker: fields.docker === 'true',
|
|
dockerReady: fields.dockerReady === 'true',
|
|
compose: fields.compose === 'true',
|
|
composeVersion: decode(fields.composeVersion),
|
|
git: fields.git === 'true',
|
|
tar: fields.tar === 'true',
|
|
checksum: fields.checksum === 'true',
|
|
baseWritable: fields.baseWritable === 'true',
|
|
};
|
|
}
|
|
|
|
class SshService {
|
|
constructor({ store, diagnostics, idleConnectionMs = 60_000, clientFactory = loadSshClient }) {
|
|
this.store = store;
|
|
this.diagnostics = diagnostics;
|
|
this.clientFactory = clientFactory;
|
|
// Every command used to pay for a TCP handshake, a key exchange and an
|
|
// authentication round trip. Sessions are kept per server for a short while
|
|
// so a sequence of commands shares one connection.
|
|
this.sessions = new Map();
|
|
this.idleConnectionMs = idleConnectionMs;
|
|
}
|
|
|
|
// A connection is only reusable for a server whose identity and credentials
|
|
// are unchanged. Anything in this key changing means a new connection.
|
|
sessionKey(server) {
|
|
return JSON.stringify([
|
|
server.id,
|
|
server.host,
|
|
server.port || 22,
|
|
server.username,
|
|
server.authType,
|
|
server.privateKeyPath || '',
|
|
server.hostFingerprint || '',
|
|
]);
|
|
}
|
|
|
|
async validateServerConfiguration(server, secrets = {}) {
|
|
if (server?.authType !== 'privateKey') return { valid: true, method: 'password' };
|
|
const privateKeyPath = String(server.privateKeyPath || '').trim();
|
|
if (!privateKeyPath) throw new Error('Select a private key file.');
|
|
const stat = await fsp.stat(privateKeyPath).catch(() => null);
|
|
if (!stat?.isFile()) {
|
|
const error = new Error(`The SSH private key file was not found: ${privateKeyPath}`);
|
|
error.code = 'SSH_PRIVATE_KEY_NOT_FOUND';
|
|
throw error;
|
|
}
|
|
const existing = server.id ? this.store.getServer(server.id) : null;
|
|
const sameKey = existing && String(existing.privateKeyPath || '') === privateKeyPath;
|
|
const storedPassphrase = sameKey ? this.store.getServerCredentials(existing.id).passphrase : '';
|
|
const passphrase = Object.prototype.hasOwnProperty.call(secrets, 'passphrase') && String(secrets.passphrase || '')
|
|
? String(secrets.passphrase)
|
|
: storedPassphrase;
|
|
const key = await fsp.readFile(privateKeyPath);
|
|
const parsed = loadSshModule().utils.parseKey(key, passphrase || undefined);
|
|
const errorResult = Array.isArray(parsed) ? parsed.find((item) => item instanceof Error) : parsed instanceof Error ? parsed : null;
|
|
if (errorResult) {
|
|
const error = new Error(`The selected file is not a usable SSH private key${passphrase ? ' with the supplied passphrase' : ''}: ${errorResult.message}`);
|
|
error.code = /encrypted|passphrase|decrypt/i.test(errorResult.message) ? 'SSH_PRIVATE_KEY_PASSPHRASE_INVALID' : 'SSH_PRIVATE_KEY_INVALID';
|
|
throw error;
|
|
}
|
|
return { valid: true, method: 'privateKey', encrypted: Boolean(passphrase), privateKeyPath };
|
|
}
|
|
|
|
async connectionOptions(server, { trustOnFirstUse = false, expectedFingerprint = null } = {}) {
|
|
const credentials = this.store.getServerCredentials(server.id);
|
|
let observedFingerprint = null;
|
|
const options = {
|
|
host: server.host,
|
|
port: server.port || 22,
|
|
username: server.username,
|
|
readyTimeout: 20_000,
|
|
keepaliveInterval: 10_000,
|
|
keepaliveCountMax: 3,
|
|
hostVerifier: (key) => {
|
|
observedFingerprint = fingerprintKey(key);
|
|
const trustedFingerprint = String(server.hostFingerprint || expectedFingerprint || '').trim();
|
|
return trustOnFirstUse || Boolean(trustedFingerprint && observedFingerprint === trustedFingerprint);
|
|
},
|
|
};
|
|
if (server.authType === 'password') options.password = credentials.password;
|
|
else {
|
|
try { options.privateKey = await fsp.readFile(server.privateKeyPath); }
|
|
catch (error) {
|
|
const wrapped = new Error(`Could not read SSH private key ${server.privateKeyPath}: ${error.message}`);
|
|
wrapped.code = 'SSH_PRIVATE_KEY_READ_FAILED';
|
|
throw wrapped;
|
|
}
|
|
if (credentials.passphrase) options.passphrase = credentials.passphrase;
|
|
}
|
|
return { options, getObservedFingerprint: () => observedFingerprint };
|
|
}
|
|
|
|
async withClient(serverId, action, options = {}) {
|
|
const server = this.store.getServer(serverId);
|
|
if (!server) throw new Error('The configured SSH server no longer exists.');
|
|
// A trust-on-first-use connection is established without checking the
|
|
// fingerprint, so it must never serve a later verified call.
|
|
if (options.trustOnFirstUse || options.expectedFingerprint) return this.withDedicatedClient(server, action, options);
|
|
return this.withPooledClient(server, action, options);
|
|
}
|
|
|
|
// Retrying is only safe while the command has not reached the server. Once a
|
|
// stream is open the remote side may already be deploying, and repeating that
|
|
// is not something this layer is allowed to decide.
|
|
isPreCommandFailure(error) {
|
|
return error?.beforeCommand === true;
|
|
}
|
|
|
|
async withPooledClient(server, action, options) {
|
|
const key = this.sessionKey(server);
|
|
for (let attempt = 0; ; attempt += 1) {
|
|
const session = await this.leaseSession(server, key, options);
|
|
try {
|
|
const result = await action(session.client, server, session.fingerprint);
|
|
this.releaseSession(session);
|
|
return result;
|
|
} catch (error) {
|
|
const staleConnection = session.reused && attempt === 0 && this.isPreCommandFailure(error);
|
|
this.discardSession(session);
|
|
if (!staleConnection) throw error;
|
|
await this.diagnostics?.debug('ssh.session.stale-retry', { serverId: server.id, host: server.host, message: error.message });
|
|
}
|
|
}
|
|
}
|
|
|
|
createSession(server, key, options) {
|
|
const entry = { key, client: null, fingerprint: null, leases: 0, dead: false, established: false, idleTimer: null, opening: null };
|
|
entry.opening = this
|
|
.withDedicatedClient(server, async (client, _server, fingerprint) => ({ client, fingerprint }), options, { keepOpen: true })
|
|
.then((opened) => {
|
|
entry.client = opened.client;
|
|
entry.fingerprint = opened.fingerprint;
|
|
entry.established = true;
|
|
// Without a standing listener an error on an idle connection is
|
|
// unhandled, which terminates the main process.
|
|
opened.client.on('error', () => this.markSessionDead(entry));
|
|
opened.client.on('close', () => this.markSessionDead(entry));
|
|
opened.client.on('end', () => this.markSessionDead(entry));
|
|
});
|
|
this.sessions.set(key, entry);
|
|
return entry;
|
|
}
|
|
|
|
async leaseSession(server, key, options) {
|
|
const pooled = this.sessions.get(key);
|
|
// Only a connection that was already up before this call may be retried on
|
|
// failure. Callers that arrive while one is still being opened share both
|
|
// the connection and its outcome.
|
|
const reused = Boolean(pooled && !pooled.dead && pooled.established);
|
|
const entry = pooled && !pooled.dead ? pooled : this.createSession(server, key, options);
|
|
entry.leases += 1;
|
|
if (entry.idleTimer) { clearTimeout(entry.idleTimer); entry.idleTimer = null; }
|
|
try {
|
|
await entry.opening;
|
|
} catch (error) {
|
|
entry.leases -= 1;
|
|
this.markSessionDead(entry);
|
|
throw error;
|
|
}
|
|
return { client: entry.client, fingerprint: entry.fingerprint, reused, entry };
|
|
}
|
|
|
|
markSessionDead(entry) {
|
|
entry.dead = true;
|
|
if (this.sessions.get(entry.key) === entry) this.sessions.delete(entry.key);
|
|
if (entry.idleTimer) { clearTimeout(entry.idleTimer); entry.idleTimer = null; }
|
|
if (entry.leases <= 0) this.endSession(entry);
|
|
}
|
|
|
|
endSession(entry) {
|
|
if (!entry.client) return;
|
|
try { entry.client.end(); } catch { /* already closed */ }
|
|
}
|
|
|
|
releaseSession(session) {
|
|
const entry = session.entry;
|
|
entry.leases -= 1;
|
|
if (entry.dead) { if (entry.leases <= 0) this.endSession(entry); return; }
|
|
if (entry.leases > 0) return;
|
|
entry.idleTimer = setTimeout(() => {
|
|
entry.idleTimer = null;
|
|
this.markSessionDead(entry);
|
|
}, this.idleConnectionMs);
|
|
entry.idleTimer.unref?.();
|
|
}
|
|
|
|
discardSession(session) {
|
|
const entry = session.entry;
|
|
entry.leases -= 1;
|
|
this.markSessionDead(entry);
|
|
}
|
|
|
|
// Closes every pooled connection. The application calls this while quitting so
|
|
// no socket outlives the process.
|
|
closeAll() {
|
|
for (const entry of [...this.sessions.values()]) {
|
|
entry.leases = 0;
|
|
this.markSessionDead(entry);
|
|
}
|
|
}
|
|
|
|
async withDedicatedClient(server, action, options = {}, { keepOpen = false } = {}) {
|
|
const serverId = server.id;
|
|
const Client = this.clientFactory();
|
|
const connection = await this.connectionOptions(server, options);
|
|
const client = new Client();
|
|
const started = Date.now();
|
|
return new Promise((resolve, reject) => {
|
|
let settled = false;
|
|
const finish = (callback, value) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
// A session that stays in the pool is closed by the pool, not here.
|
|
if (!(keepOpen && callback === resolve)) { try { client.end(); } catch { /* already closed */ } }
|
|
callback(value);
|
|
};
|
|
client.once('ready', async () => {
|
|
try {
|
|
const data = await action(client, server, connection.getObservedFingerprint());
|
|
await this.diagnostics?.debug('ssh.connection.completed', { serverId, host: server.host, durationMs: Date.now() - started });
|
|
finish(resolve, data);
|
|
} catch (error) { finish(reject, error); }
|
|
});
|
|
// Deliberately not `once`: a connection that already failed can emit a
|
|
// second error while it is being torn down, and an unhandled 'error' event
|
|
// on an EventEmitter terminates the main process.
|
|
client.on('error', async (error) => {
|
|
if (settled) return;
|
|
const observed = connection.getObservedFingerprint();
|
|
const mismatch = Boolean(server.hostFingerprint && observed && server.hostFingerprint !== observed);
|
|
const wrapped = new Error(mismatch
|
|
? `SSH host identity changed. Expected ${server.hostFingerprint}, but the server presented ${observed}.`
|
|
: `SSH connection failed: ${error.message}`);
|
|
wrapped.code = mismatch ? 'SSH_HOST_KEY_MISMATCH' : (error.code || 'SSH_CONNECTION_FAILED');
|
|
wrapped.expectedFingerprint = mismatch ? server.hostFingerprint : undefined;
|
|
wrapped.observedFingerprint = mismatch ? observed : undefined;
|
|
await this.diagnostics?.warning('ssh.connection.failed', { serverId, host: server.host, durationMs: Date.now() - started, code: wrapped.code, message: wrapped.message });
|
|
finish(reject, wrapped);
|
|
});
|
|
client.connect(connection.options);
|
|
});
|
|
}
|
|
|
|
execClient(client, command, { timeout = 15 * 60_000, maxOutput = 2 * 1024 * 1024 } = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
let completed = false;
|
|
const timer = setTimeout(() => {
|
|
if (completed) return;
|
|
completed = true;
|
|
reject(new Error('The SSH command timed out.'));
|
|
}, timeout);
|
|
client.exec(command, (error, stream) => {
|
|
if (error) {
|
|
clearTimeout(timer);
|
|
completed = true;
|
|
// The channel never opened, so the command did not reach the server.
|
|
// This is the only failure the pool is allowed to retry.
|
|
error.beforeCommand = true;
|
|
reject(error);
|
|
return;
|
|
}
|
|
let stdout = '';
|
|
let stderr = '';
|
|
let stdoutBytes = 0;
|
|
let stderrBytes = 0;
|
|
let truncated = false;
|
|
const append = (target, chunk) => {
|
|
const text = chunk.toString();
|
|
const bytes = Buffer.byteLength(text);
|
|
if (target === 'stdout') {
|
|
if (stdoutBytes + bytes <= maxOutput) stdout += text;
|
|
else truncated = true;
|
|
stdoutBytes += bytes;
|
|
} else {
|
|
if (stderrBytes + bytes <= maxOutput) stderr += text;
|
|
else truncated = true;
|
|
stderrBytes += bytes;
|
|
}
|
|
};
|
|
stream.on('data', (chunk) => append('stdout', chunk));
|
|
stream.stderr.on('data', (chunk) => append('stderr', chunk));
|
|
stream.on('close', (code, signal) => {
|
|
if (completed) return;
|
|
completed = true;
|
|
clearTimeout(timer);
|
|
if (truncated) {
|
|
const failure = new Error(`Remote command output exceeded the ${maxOutput}-byte safety limit. ForgeFlow refused to use an incomplete result.`);
|
|
failure.code = 'SSH_OUTPUT_TRUNCATED';
|
|
failure.stdoutBytes = stdoutBytes;
|
|
failure.stderrBytes = stderrBytes;
|
|
reject(failure);
|
|
} else if (code !== 0) {
|
|
const failure = new Error(`Remote command failed with exit code ${code}: ${(stderr || stdout).trim().slice(-4000)}`);
|
|
failure.code = 'SSH_COMMAND_FAILED';
|
|
failure.exitCode = code;
|
|
failure.signal = signal;
|
|
reject(failure);
|
|
} else resolve({ stdout, stderr, exitCode: code, truncated: false });
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
ensureUploadTarget(target) {
|
|
const normalized = String(target || '').replace(/\\/g, '/');
|
|
if (!normalized.startsWith('/') || normalized.includes('\0') || normalized.split('/').includes('..')) throw new Error('Remote upload path must be an absolute safe Unix path.');
|
|
return normalized;
|
|
}
|
|
|
|
async withSftp(serverId, remotePath, action) {
|
|
const server = this.store.getServer(serverId);
|
|
if (!server?.hostFingerprint) {
|
|
const error = new Error('Test and trust the SSH server fingerprint before uploading deployment assets.');
|
|
error.code = 'SSH_HOST_NOT_TRUSTED';
|
|
throw error;
|
|
}
|
|
const target = this.ensureUploadTarget(remotePath);
|
|
return this.withClient(serverId, (client) => new Promise((resolve, reject) => {
|
|
client.sftp((sftpError, sftp) => {
|
|
if (sftpError) { reject(sftpError); return; }
|
|
const parts = path.dirname(target).split('/').filter(Boolean);
|
|
let current = '';
|
|
const ensureNext = (index) => {
|
|
if (index >= parts.length) {
|
|
Promise.resolve(action(sftp, target)).then(resolve, reject);
|
|
return;
|
|
}
|
|
current += `/${parts[index]}`;
|
|
sftp.stat(current, (statError, attributes) => {
|
|
if (!statError) {
|
|
if (typeof attributes?.isDirectory === 'function' && !attributes.isDirectory()) { reject(new Error(`Remote upload parent exists but is not a directory: ${current}`)); return; }
|
|
ensureNext(index + 1);
|
|
return;
|
|
}
|
|
if (![2, 'ENOENT'].includes(statError.code)) { reject(statError); return; }
|
|
sftp.mkdir(current, { mode: 0o755 }, (mkdirError) => {
|
|
if (!mkdirError) { ensureNext(index + 1); return; }
|
|
sftp.stat(current, (retryError, retryAttributes) => {
|
|
if (!retryError && (typeof retryAttributes?.isDirectory !== 'function' || retryAttributes.isDirectory())) ensureNext(index + 1);
|
|
else reject(mkdirError);
|
|
});
|
|
});
|
|
});
|
|
};
|
|
ensureNext(0);
|
|
});
|
|
}), { trustOnFirstUse: false });
|
|
}
|
|
|
|
async uploadBuffer(serverId, remotePath, content, { mode = 0o600 } = {}) {
|
|
const data = Buffer.isBuffer(content) ? content : Buffer.from(content);
|
|
return this.withSftp(serverId, remotePath, (sftp, target) => new Promise((resolve, reject) => {
|
|
const stream = sftp.createWriteStream(target, { mode });
|
|
stream.once('error', reject);
|
|
stream.once('close', () => resolve({ remotePath: target, size: data.length }));
|
|
stream.end(data);
|
|
}));
|
|
}
|
|
|
|
async uploadFile(serverId, localPath, remotePath, { mode = 0o600, onProgress = null } = {}) {
|
|
const stat = await fsp.stat(localPath);
|
|
if (!stat.isFile()) throw new Error(`Local upload source is not a file: ${localPath}`);
|
|
return this.withSftp(serverId, remotePath, (sftp, target) => new Promise((resolve, reject) => {
|
|
const options = {
|
|
mode,
|
|
step: (totalTransferred, _chunk, total) => onProgress?.({ transferred: totalTransferred, total: total || stat.size }),
|
|
};
|
|
sftp.fastPut(localPath, target, options, (error) => {
|
|
if (error) { reject(error); return; }
|
|
resolve({ remotePath: target, size: stat.size });
|
|
});
|
|
}));
|
|
}
|
|
|
|
async probeHostFingerprint(serverId) {
|
|
const server = this.store.getServer(serverId);
|
|
if (!server) throw new Error('The configured SSH server no longer exists.');
|
|
const Client = this.clientFactory();
|
|
const client = new Client();
|
|
let observedFingerprint = null;
|
|
return new Promise((resolve, reject) => {
|
|
let settled = false;
|
|
const finish = (callback, value) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
try { client.end(); } catch { /* handshake already closed */ }
|
|
callback(value);
|
|
};
|
|
const completeProbe = (error = null) => {
|
|
if (observedFingerprint) {
|
|
finish(resolve, {
|
|
fingerprint: observedFingerprint,
|
|
server: { id: server.id, name: server.name, host: server.host, port: server.port || 22 },
|
|
});
|
|
return;
|
|
}
|
|
const wrapped = new Error(`Could not read the SSH host fingerprint: ${error?.message || 'the server closed the handshake'}`);
|
|
wrapped.code = error?.code || 'SSH_HOST_KEY_PROBE_FAILED';
|
|
finish(reject, wrapped);
|
|
};
|
|
const timer = setTimeout(() => completeProbe(new Error('The SSH host-key probe timed out.')), 25_000);
|
|
client.on('error', completeProbe);
|
|
client.on('close', () => completeProbe());
|
|
client.on('end', () => completeProbe());
|
|
client.connect({
|
|
host: server.host,
|
|
port: server.port || 22,
|
|
username: server.username,
|
|
readyTimeout: 20_000,
|
|
hostVerifier: (key) => {
|
|
observedFingerprint = fingerprintKey(key);
|
|
return false;
|
|
},
|
|
});
|
|
});
|
|
}
|
|
|
|
async test(serverId, { trustOnFirstUse = false, expectedFingerprint = null } = {}) {
|
|
return this.withClient(serverId, async (client, server, fingerprint) => {
|
|
const script = `
|
|
platform=$(uname -srm 2>/dev/null || true)
|
|
docker=false; docker_ready=false; compose=false; compose_version=''; git=false; tar_ok=false; checksum=false; base_writable=false
|
|
command -v docker >/dev/null 2>&1 && docker=true
|
|
[ "$docker" = true ] && docker info >/dev/null 2>&1 && docker_ready=true
|
|
if [ "$docker" = true ]; then
|
|
if docker compose version >/dev/null 2>&1; then compose=true; compose_version=$(docker compose version 2>/dev/null | head -n1); elif command -v docker-compose >/dev/null 2>&1; then compose=true; compose_version=$(docker-compose version 2>/dev/null | head -n1); fi
|
|
fi
|
|
command -v git >/dev/null 2>&1 && git=true
|
|
command -v tar >/dev/null 2>&1 && tar_ok=true
|
|
(command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1) && checksum=true
|
|
base=${shellQuote(server.basePath)}
|
|
if [ -d "$base" ]; then [ -w "$base" ] && base_writable=true; else parent=$(dirname "$base"); [ -d "$parent" ] && [ -w "$parent" ] && base_writable=true; fi
|
|
printf '__FORGEFLOW_SERVER_TEST__\\n'
|
|
printf 'platform=%s\\n' "$(printf '%s' "$platform" | base64 | tr -d '\\r\\n')"
|
|
printf 'docker=%s\\n' "$docker"
|
|
printf 'dockerReady=%s\\n' "$docker_ready"
|
|
printf 'compose=%s\\n' "$compose"
|
|
printf 'composeVersion=%s\\n' "$(printf '%s' "$compose_version" | base64 | tr -d '\\r\\n')"
|
|
printf 'git=%s\\n' "$git"
|
|
printf 'tar=%s\\n' "$tar_ok"
|
|
printf 'checksum=%s\\n' "$checksum"
|
|
printf 'baseWritable=%s\\n' "$base_writable"
|
|
`;
|
|
const result = await this.execClient(client, script, { timeout: 30_000, maxOutput: 256 * 1024 });
|
|
const capabilities = parseCapabilityOutput(result.stdout);
|
|
return {
|
|
connected: true,
|
|
fingerprint,
|
|
server: { id: server.id, name: server.name, host: server.host, basePath: server.basePath },
|
|
capabilities,
|
|
output: [capabilities.platform, capabilities.composeVersion].filter(Boolean).join('\n'),
|
|
};
|
|
}, { trustOnFirstUse, expectedFingerprint });
|
|
}
|
|
|
|
async exec(serverId, command, options = {}) {
|
|
const server = this.store.getServer(serverId);
|
|
if (!server?.hostFingerprint) {
|
|
const error = new Error('Test and trust the SSH server fingerprint before running deployment commands.');
|
|
error.code = 'SSH_HOST_NOT_TRUSTED';
|
|
throw error;
|
|
}
|
|
return this.withClient(serverId, (client) => this.execClient(client, command, options), { trustOnFirstUse: false });
|
|
}
|
|
}
|
|
|
|
module.exports = { SshService, shellQuote, fingerprintKey, parseCapabilityOutput };
|