From f8b97a4f338e94aa68c1a7a5b8aba8faea836f1d Mon Sep 17 00:00:00 2001 From: "Matthew Bryant (mandatory)" Date: Sat, 6 Nov 2021 13:49:12 -0700 Subject: [PATCH] Fixes some issues with certificate generation causing requests to fail and also *hopefully* fixes the proxy auth errors --- .gitignore | 2 + anyproxy/lib/certMgr.js | 2 +- anyproxy/lib/node-easy-cert/certGenerator.js | 126 ++++++++++ anyproxy/lib/node-easy-cert/errorConstants.js | 12 + anyproxy/lib/node-easy-cert/index.js | 235 ++++++++++++++++++ anyproxy/lib/node-easy-cert/util.js | 42 ++++ anyproxy/lib/node-easy-cert/winCertUtil.js | 39 +++ anyproxy/lib/requestHandler.js | 28 ++- anyproxy/proxy.js | 2 + package.json | 5 +- 10 files changed, 480 insertions(+), 13 deletions(-) create mode 100644 anyproxy/lib/node-easy-cert/certGenerator.js create mode 100644 anyproxy/lib/node-easy-cert/errorConstants.js create mode 100644 anyproxy/lib/node-easy-cert/index.js create mode 100644 anyproxy/lib/node-easy-cert/util.js create mode 100644 anyproxy/lib/node-easy-cert/winCertUtil.js diff --git a/.gitignore b/.gitignore index 93ccfda..6fdc121 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules/* .DS_Store *.key *.crt +*.env +dev.sh \ No newline at end of file diff --git a/anyproxy/lib/certMgr.js b/anyproxy/lib/certMgr.js index d06cccc..8fb0b99 100644 --- a/anyproxy/lib/certMgr.js +++ b/anyproxy/lib/certMgr.js @@ -1,6 +1,6 @@ 'use strict' -const EasyCert = require('node-easy-cert'); +const EasyCert = require('./node-easy-cert/index.js'); const co = require('co'); const os = require('os'); const inquirer = require('inquirer'); diff --git a/anyproxy/lib/node-easy-cert/certGenerator.js b/anyproxy/lib/node-easy-cert/certGenerator.js new file mode 100644 index 0000000..d468838 --- /dev/null +++ b/anyproxy/lib/node-easy-cert/certGenerator.js @@ -0,0 +1,126 @@ +'use strict' + +const forge = require('node-forge'); +const Util = require('./util'); + +let defaultAttrs = [ + { name: 'countryName', value: 'CN' }, + { name: 'organizationName', value: 'EasyCert' }, + { shortName: 'ST', value: 'SH' }, + { shortName: 'OU', value: 'EasyCert SSL' } +]; + +/** +* different domain format needs different SAN +* +*/ +function getExtensionSAN(domain = '') { + const isIpDomain = Util.isIpDomain(domain); + if (isIpDomain) { + return { + name: 'subjectAltName', + altNames: [{ type: 7, ip: domain }] + }; + } else { + return { + name: 'subjectAltName', + altNames: [{ type: 2, value: domain }] + }; + } +} + +function getKeysAndCert(serialNumber) { + const keys = forge.pki.rsa.generateKeyPair(2048); + const cert = forge.pki.createCertificate(); + cert.publicKey = keys.publicKey; + cert.serialNumber = (Math.floor(Math.random() * 100000) + ''); + console.log(`serial #${cert.serialNumber}`) + var now = Date.now(); + // compatible with apple's updated cert policy: https://support.apple.com/en-us/HT210176 + cert.validity.notBefore = new Date(now - 24 * 60 * 60 * 1000); // 1 day before + cert.validity.notAfter = new Date(now + 824 * 24 * 60 * 60 * 1000); // 824 days after + return { + keys, + cert + }; +} + +function generateRootCA(commonName) { + const keysAndCert = getKeysAndCert(); + const keys = keysAndCert.keys; + const cert = keysAndCert.cert; + + commonName = commonName || 'CertManager'; + + const attrs = defaultAttrs.concat([ + { + name: 'commonName', + value: commonName + } + ]); + cert.setSubject(attrs); + cert.setIssuer(attrs); + cert.setExtensions([ + { name: 'basicConstraints', cA: true }, + // { name: 'keyUsage', keyCertSign: true, digitalSignature: true, nonRepudiation: true, keyEncipherment: true, dataEncipherment: true }, + // { name: 'extKeyUsage', serverAuth: true, clientAuth: true, codeSigning: true, emailProtection: true, timeStamping: true }, + // { name: 'nsCertType', client: true, server: true, email: true, objsign: true, sslCA: true, emailCA: true, objCA: true }, + // { name: 'subjectKeyIdentifier' } + ]); + + cert.sign(keys.privateKey, forge.md.sha256.create()); + + return { + privateKey: forge.pki.privateKeyToPem(keys.privateKey), + publicKey: forge.pki.publicKeyToPem(keys.publicKey), + certificate: forge.pki.certificateToPem(cert) + }; +} + +function generateCertsForHostname(domain, rootCAConfig) { + // generate a serialNumber for domain + const md = forge.md.md5.create(); + md.update(domain); + + const keysAndCert = getKeysAndCert(md.digest().toHex()); + const keys = keysAndCert.keys; + const cert = keysAndCert.cert; + + const caCert = forge.pki.certificateFromPem(rootCAConfig.cert); + const caKey = forge.pki.privateKeyFromPem(rootCAConfig.key); + + // issuer from CA + cert.setIssuer(caCert.subject.attributes); + + const attrs = defaultAttrs.concat([ + { + name: 'commonName', + value: domain + } + ]); + + const extensions = [ + { name: 'basicConstraints', cA: false }, + getExtensionSAN(domain) + ]; + + cert.setSubject(attrs); + cert.setExtensions(extensions); + + cert.sign(caKey, forge.md.sha256.create()); + + return { + privateKey: forge.pki.privateKeyToPem(keys.privateKey), + publicKey: forge.pki.publicKeyToPem(keys.publicKey), + certificate: forge.pki.certificateToPem(cert) + }; +} + +// change the default attrs +function setDefaultAttrs(attrs) { + defaultAttrs = attrs; +} + +module.exports.generateRootCA = generateRootCA; +module.exports.generateCertsForHostname = generateCertsForHostname; +module.exports.setDefaultAttrs = setDefaultAttrs; diff --git a/anyproxy/lib/node-easy-cert/errorConstants.js b/anyproxy/lib/node-easy-cert/errorConstants.js new file mode 100644 index 0000000..1e04680 --- /dev/null +++ b/anyproxy/lib/node-easy-cert/errorConstants.js @@ -0,0 +1,12 @@ +/** +* Map all the error code here +* +*/ + +'use strict'; + +module.exports = { + ROOT_CA_NOT_EXISTS: 'ROOT_CA_NOT_EXISTS', // root CA has not been generated yet + ROOT_CA_EXISTED: 'ROOT_CA_EXISTED', // root CA was existed, be ware that it will be overwrited + ROOT_CA_COMMON_NAME_UNSPECIFIED: 'ROOT_CA_COMMON_NAME_UNSPECIFIED' // commonName for rootCA is required +}; diff --git a/anyproxy/lib/node-easy-cert/index.js b/anyproxy/lib/node-easy-cert/index.js new file mode 100644 index 0000000..19fe213 --- /dev/null +++ b/anyproxy/lib/node-easy-cert/index.js @@ -0,0 +1,235 @@ +'use strict' + +delete require.cache['./certGenerator']; + +const path = require('path'), + fs = require('fs'), + color = require('colorful'), + certGenerator = require('./certGenerator'), + util = require('./util'), + Errors = require('./errorConstants'), + https = require('https'), + AsyncTask = require('async-task-mgr'), + winCertUtil = require('./winCertUtil'), + exec = require('child_process').exec; + +const DOMAIN_TO_VERIFY_HTTPS = 'localtest.me'; + +function getPort() { + return new Promise((resolve, reject) => { + const server = require('net').createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, () => { + const port = server.address().port; + server.close(() => { + resolve(port); + }); + }); + }); +} + +function CertManager(options) { + options = options || {}; + const rootDirName = util.getDefaultRootDirName(); + const rootDirPath = options.rootDirPath || path.join(util.getUserHome(), '/' + rootDirName + '/'); + + if (options.defaultCertAttrs) { + certGenerator.setDefaultAttrs(options.defaultCertAttrs); + } + + const certDir = rootDirPath, + rootCAcrtFilePath = path.join(certDir, 'rootCA.crt'), + rootCAkeyFilePath = path.join(certDir, 'rootCA.key'), + createCertTaskMgr = new AsyncTask(); + let cacheRootCACrtFileContent, + cacheRootCAKeyFileContent; + let rootCAExists = false; + + if (!fs.existsSync(certDir)) { + try { + fs.mkdirSync(certDir, '0777'); + } catch (e) { + console.log('==========='); + console.log('failed to create cert dir ,please create one by yourself - ' + certDir); + console.log('==========='); + } + } + + function getCertificate(hostname, certCallback) { + if (!_checkRootCA()) { + console.log(color.yellow('please generate root CA before getting certificate for sub-domains')); + certCallback && certCallback(Errors.ROOT_CA_NOT_EXISTS); + return; + } + const keyFile = path.join(certDir, '__hostname.key'.replace(/__hostname/, hostname)), + crtFile = path.join(certDir, '__hostname.crt'.replace(/__hostname/, hostname)); + + if (!cacheRootCACrtFileContent || !cacheRootCAKeyFileContent) { + cacheRootCACrtFileContent = fs.readFileSync(rootCAcrtFilePath, { encoding: 'utf8' }); + cacheRootCAKeyFileContent = fs.readFileSync(rootCAkeyFilePath, { encoding: 'utf8' }); + } + + createCertTaskMgr.addTask(hostname, (callback) => { + if (!fs.existsSync(keyFile) || !fs.existsSync(crtFile)) { + try { + const result = certGenerator.generateCertsForHostname(hostname, { + cert: cacheRootCACrtFileContent, + key: cacheRootCAKeyFileContent + }); + fs.writeFileSync(keyFile, result.privateKey); + fs.writeFileSync(crtFile, result.certificate); + callback(null, result.privateKey, result.certificate); + } catch (e) { + callback(e); + } + } else { + callback(null, fs.readFileSync(keyFile), fs.readFileSync(crtFile)); + } + }, (err, keyContent, crtContent) => { + if (!err) { + certCallback(null, keyContent, crtContent); + } else { + certCallback(err); + } + }); + } + + function clearCerts(cb) { + util.deleteFolderContentsRecursive(certDir); + cb && cb(); + } + + function isRootCAFileExists() { + return (fs.existsSync(rootCAcrtFilePath) && fs.existsSync(rootCAkeyFilePath)); + } + + function generateRootCA(config, certCallback) { + if (!config || !config.commonName) { + console.error(color.red('The "config.commonName" for rootCA is required, please specify.')); + certCallback(Errors.ROOT_CA_COMMON_NAME_UNSPECIFIED); + return; + } + + if (isRootCAFileExists()) { + if (config.overwrite) { + startGenerating(config.commonName, certCallback); + } else { + console.error(color.red('The rootCA exists already, if you want to overwrite it, please specify the "config.overwrite=true"')); + certCallback(Errors.ROOT_CA_EXISTED); + } + } else { + startGenerating(config.commonName, certCallback); + } + + function startGenerating(commonName, cb) { + // clear old certs + clearCerts(() => { + console.log(color.green('temp certs cleared')); + try { + const result = certGenerator.generateRootCA(commonName); + fs.writeFileSync(rootCAkeyFilePath, result.privateKey); + fs.writeFileSync(rootCAcrtFilePath, result.certificate); + + console.log(color.green('rootCA generated')); + console.log(color.green(color.bold('PLEASE TRUST the rootCA.crt in ' + certDir))); + + cb && cb(null, rootCAkeyFilePath, rootCAcrtFilePath); + } catch (e) { + console.log(color.red(e)); + console.log(color.red(e.stack)); + console.log(color.red('fail to generate root CA')); + cb && cb(e); + } + }); + } + } + + function getRootCAFilePath() { + return isRootCAFileExists() ? rootCAcrtFilePath : ''; + } + + function getRootDirPath() { + return rootDirPath; + } + + function _checkRootCA() { + if (rootCAExists) { + return true; + } + + if (!isRootCAFileExists()) { + console.log(color.red('can not find rootCA.crt or rootCA.key')); + console.log(color.red('you may generate one')); + return false; + } else { + rootCAExists = true; + return true; + } + } + + function ifRootCATrusted(callback) { + if (!isRootCAFileExists()) { + callback && callback(new Error('ROOTCA_NOT_EXIST')); + } else if (/^win/.test(process.platform)) { + winCertUtil.ifWinRootCATrusted() + .then((ifTrusted) => { + callback && callback(null, ifTrusted) + }) + .catch((e) => { + callback && callback(null, false); + }) + } else { + const HTTPS_RESPONSE = 'HTTPS Server is ON'; + // localtest.me --> 127.0.0.1 + getCertificate(DOMAIN_TO_VERIFY_HTTPS, (e, key, cert) => { + getPort() + .then(port => { + if (e) { + callback && callback(e); + return; + } + const server = https + .createServer( + { + ca: fs.readFileSync(rootCAcrtFilePath), + key, + cert + }, + (req, res) => { + res.end(HTTPS_RESPONSE); + } + ) + .listen(port); + + // do not use node.http to test the cert. Ref: https://github.com/nodejs/node/issues/4175 + const testCmd = `curl https://${DOMAIN_TO_VERIFY_HTTPS}:${port}`; + exec(testCmd, { timeout: 1000 }, (error, stdout, stderr) => { + server.close(); + if (error) { + callback && callback(null, false); + } + if (stdout && stdout.indexOf(HTTPS_RESPONSE) >= 0) { + callback && callback(null, true); + } else { + callback && callback(null, false); + } + }); + }) + .catch(callback); + }); + } + } + + return { + getRootCAFilePath, + generateRootCA, + getCertificate, + clearCerts, + isRootCAFileExists, + ifRootCATrusted, + getRootDirPath, + }; +} + +module.exports = CertManager; diff --git a/anyproxy/lib/node-easy-cert/util.js b/anyproxy/lib/node-easy-cert/util.js new file mode 100644 index 0000000..c0b07af --- /dev/null +++ b/anyproxy/lib/node-easy-cert/util.js @@ -0,0 +1,42 @@ +'use strict' + +const fs = require('fs'); +const path = require('path'); + +function deleteFolderContentsRecursive(dirPath) { + if (!dirPath.trim() || dirPath === '/') { + throw new Error('can_not_delete_this_dir'); + } + + if (fs.existsSync(dirPath)) { + fs.readdirSync(dirPath).forEach((file, index) => { + const curPath = path.join(dirPath, file); + if (fs.lstatSync(curPath).isDirectory()) { + deleteFolderContentsRecursive(curPath); + } else { // delete all files + fs.unlinkSync(curPath); + } + }); + // keep the folder + // fs.rmdirSync(dirPath); + } +} + +module.exports.getUserHome = function () { + return process.env.HOME || process.env.USERPROFILE; +}; + +module.exports.getDefaultRootDirName = function () { + return '.node_easy_certs'; +}; + +/* +* identify whether the +*/ +module.exports.isIpDomain = function (domain = '') { + const ipReg = /^\d+?\.\d+?\.\d+?\.\d+?$/; + + return ipReg.test(domain); +}; + +module.exports.deleteFolderContentsRecursive = deleteFolderContentsRecursive; diff --git a/anyproxy/lib/node-easy-cert/winCertUtil.js b/anyproxy/lib/node-easy-cert/winCertUtil.js new file mode 100644 index 0000000..ddabc7c --- /dev/null +++ b/anyproxy/lib/node-easy-cert/winCertUtil.js @@ -0,0 +1,39 @@ +/** +* +*/ +const Shell = require('node-powershell'); + +const anyProxyCertReg = /CN=AnyProxy,\sOU=AnyProxy\sSSL\sProxy/; + +/** + * detect whether root CA is trusted + */ +function ifWinRootCATrusted() { + const ps = new Shell({ + executionPolicy: 'Bypass', + debugMsg: false, + noProfile: true + }); + + return new Promise((resolve, reject) => { + ps.addCommand('Get-ChildItem', [ + { + name: 'path', + value: 'cert:\\CurrentUser\\Root' + } + ]); + ps.invoke() + .then((output) => { + const isCATrusted = anyProxyCertReg.test(output); + ps.dispose(); + resolve(isCATrusted); + }) + .catch((err) => { + console.log(err); + ps.dispose(); + resolve(false); + }); + }) +} + +module.exports.ifWinRootCATrusted = ifWinRootCATrusted; diff --git a/anyproxy/lib/requestHandler.js b/anyproxy/lib/requestHandler.js index 1fbb950..f226266 100644 --- a/anyproxy/lib/requestHandler.js +++ b/anyproxy/lib/requestHandler.js @@ -593,7 +593,14 @@ function getConnectReqHandler(userRule, recorder, httpsServerMgr) { .then(() => { return new Promise((resolve) => { // mark socket connection as established, to detect the request protocol - cltSocket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', resolve); + if(proxy_authorization) { + cltSocket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', resolve); + } else { + cltSocket.end('HTTP/' + req.httpVersion + ' 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="Please provide your credentials."\r\n\r\n', 'UTF-8'); + resolve(); + return + } + }); }) .then(() => { @@ -682,24 +689,25 @@ function getConnectReqHandler(userRule, recorder, httpsServerMgr) { return new Promise((resolve, reject) => { const conn = net.connect(serverInfo.port, serverInfo.host, () => { - //throttle for direct-foward https - if (global._throttle && !shouldIntercept) { - requestStream.pipe(conn); - conn.pipe(global._throttle.throttle()).pipe(cltSocket); - } else { - requestStream.pipe(conn); - conn.pipe(cltSocket); - } - if(!global.proxyAuthPassthru) { global.proxyAuthPassthru = {}; } //console.log(`Source port for HTTPS -> HTTP: ${conn.localPort}`); + //console.log(`Setting proxy auth creds in global map... lport: ${conn.localPort} | value: ${proxy_authorization}`); global.proxyAuthPassthru[conn.localPort] = { 'proxy_authorization': proxy_authorization }; + //throttle for direct-foward https + if (global._throttle && !shouldIntercept) { + requestStream.pipe(conn); + conn.pipe(global._throttle.throttle()).pipe(cltSocket); + } else { + requestStream.pipe(conn); + conn.pipe(cltSocket); + } + resolve(); }); diff --git a/anyproxy/proxy.js b/anyproxy/proxy.js index 1230a07..9d5a005 100644 --- a/anyproxy/proxy.js +++ b/anyproxy/proxy.js @@ -14,6 +14,8 @@ const http = require('http'), wsServerMgr = require('./lib/wsServerMgr'), ThrottleGroup = require('stream-throttle').ThrottleGroup; + +events.EventEmitter.prototype._maxListeners = 100; // const memwatch = require('memwatch-next'); // setInterval(() => { diff --git a/package.json b/package.json index 4b6434b..63d5aee 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "moment": "^2.24.0", "nedb": "^1.8.0", "node-cache": "^5.1.0", - "node-easy-cert": "^1.0.0", "pg": "^7.18.2", "pug": "^2.0.0-beta6", "qrcode-npm": "0.0.3", @@ -49,6 +48,8 @@ "thunkify": "^2.1.2", "uuid": "^7.0.2", "whatwg-fetch": "^1.0.0", - "ws": "^5.1.0" + "ws": "^5.1.0", + "node-forge": "^0.6.42", + "node-powershell": "^3.3.1" } }