MITM module and MITM support

README.md

@@ -367,9 +367,11 @@ Consult the Java API docs for more info.
 
 ### SSL Support
 
-**LittleProxy support for MITM:** In the current beta release, the `browsermob-core-littleproxy` module supports MITM but does not support dynamic certificate spoofing. In most cases this will not affect your tests, but browsers accessing HTTPS websites through BrowserMob Proxy will be notified that the certificates cannot be verified.
+**BrowserMob with LittleProxy now supports full MITM:** For most users, MITM will work out-of-the-box with default settings. Install the [ca-certificate-rsa.cer](/sslSupport/ca-certificate-rsa.cer) file in your browser or HTTP client to avoid untrusted certificate warnings. Generally, it is safer to generate your own private key, rather than using the .cer files distributed with BrowserMob Proxy. See the [README file in the `mitm` module](/mitm/README.md) for instructions on generating or using your own root certificate and private key with MITM.
 
-**Legacy Jetty-based support for MITM:** The legacy `ProxyServer` implementation using the `browsermob-core` module supports MITM and dynamic certificate spoofing. To avoid browser certificate warnings, a Certificate Authority must be installed in the browser. This allows the browser to trust all the SSL traffic coming from the proxy, which will be proxied using a classic man-in-the1-middle technique. IT IS CRITICAL THAT YOU NOT INSTALL THIS CERTIFICATE AUTHORITY ON A BROWSER THAT IS USED FOR ANYTHING OTHER THAN TESTING.
+**Legacy Jetty-based ProxyServer support for MITM:** As of version 2.1.0-beta-4, the legacy `ProxyServer` implementation uses the same `ca-certificate-rsa.cer` root certificate as the LittleProxy implementation. The previous cybervillainsCA.cer certificate has been removed.
+
+**Note: DO NOT** permanently install the .cer files distributed with BrowserMob Proxy in users' browsers. They should be used for testing only and must not be used with general web browsing.
 
 If you're doing testing with Selenium, you'll want to make sure that the browser profile that gets set up by Selenium not only has the proxy configured, but also has the CA installed. Unfortuantely, there is no API for doing this in Selenium, so you'll have to solve it uniquely for each browser type. We hope to make this easier in upcoming releases.
 

browsermob-core-littleproxy/pom.xml

@@ -266,6 +266,22 @@
             <artifactId>netty-all</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk15on</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>net.lightbody.bmp</groupId>
+            <artifactId>mitm</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <!-- Due to usage in LegacyProxyServer and BrowserMobProxyServer, this dependency needs to be given "provided" scope.
                      It is not used by the BMP LittleProxy implementation itself. -->
         <dependency>

browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java

@@ -29,6 +29,9 @@
 import net.lightbody.bmp.filters.RewriteUrlFilter;
 import net.lightbody.bmp.filters.UnregisterRequestFilter;
 import net.lightbody.bmp.filters.WhitelistFilter;
+import net.lightbody.bmp.mitm.KeyStoreFileCertificateSource;
+import net.lightbody.bmp.mitm.keys.RSAKeyGenerator;
+import net.lightbody.bmp.mitm.manager.ImpersonatingMitmManager;
 import net.lightbody.bmp.proxy.ActivityMonitor;
 import net.lightbody.bmp.proxy.BlacklistEntry;
 import net.lightbody.bmp.proxy.CaptureType;
@@ -41,7 +44,6 @@
 import net.lightbody.bmp.proxy.http.RequestInterceptor;
 import net.lightbody.bmp.proxy.http.ResponseInterceptor;
 import net.lightbody.bmp.proxy.util.BrowserMobProxyUtil;
-import net.lightbody.bmp.ssl.BrowserMobProxyMitmManager;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.HttpResponseInterceptor;
 import org.java_bandwidthlimiter.StreamManager;
@@ -53,6 +55,7 @@
 import org.littleshoot.proxy.HttpFiltersSourceAdapter;
 import org.littleshoot.proxy.HttpProxyServer;
 import org.littleshoot.proxy.HttpProxyServerBootstrap;
+import org.littleshoot.proxy.MitmManager;
 import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
 import org.littleshoot.proxy.impl.ProxyUtils;
 import org.littleshoot.proxy.impl.ThreadPoolConfiguration;
@@ -88,7 +91,13 @@ public class BrowserMobProxyServer implements BrowserMobProxy, LegacyProxyServer
     private static final Logger log = LoggerFactory.getLogger(BrowserMobProxyServer.class);
 
     //TODO: extract the version string into a more suitable location
-    private static final HarNameVersion HAR_CREATOR_VERSION = new HarNameVersion("BrowserMob Proxy", "2.1.0-beta-3-littleproxy");
+    private static final HarNameVersion HAR_CREATOR_VERSION = new HarNameVersion("BrowserMob Proxy", "2.1.0-beta-4-littleproxy");
+
+    /* Default MITM resources */
+    private static final String KEYSTORE_RESOURCE = "/sslSupport/ca-keystore-rsa.p12";
+    private static final String KEYSTORE_TYPE = "PKCS12";
+    private static final String KEYSTORE_PRIVATE_KEY_ALIAS = "key";
+    private static final String KEYSTORE_PASSWORD = "password";
 
     /**
      * The default pseudonym to use when adding the Via header to proxied requests.
@@ -115,6 +124,11 @@ public class BrowserMobProxyServer implements BrowserMobProxy, LegacyProxyServer
      */
     private volatile boolean mitmDisabled = false;
 
+    /**
+     * The MITM manager that will be used for HTTPS requests.
+     */
+    private volatile MitmManager mitmManager;
+
     /**
      * The list of filterFactories that will generate the filters that implement browsermob-proxy behavior.
      */
@@ -226,6 +240,11 @@ public class BrowserMobProxyServer implements BrowserMobProxy, LegacyProxyServer
      */
     private volatile boolean errorOnUnsupportedOperation = false;
 
+    /**
+     * When true, will not validate upstream servers' certificates. Currently only applicable when MITMing.
+     */
+    private volatile boolean trustAllServers = true;
+
     /**
      * Resolver to use when resolving hostnames to IP addresses. This is a bridge between {@link org.littleshoot.proxy.HostResolver} and
      * {@link net.lightbody.bmp.proxy.dns.AdvancedHostResolver}. It allows the resolvers to be changed on-the-fly without re-bootstrapping the
@@ -345,7 +364,15 @@ public int getMaximumResponseBufferSizeInBytes() {
 
 
         if (!mitmDisabled) {
-            bootstrap.withManInTheMiddle(new BrowserMobProxyMitmManager());
+            if (mitmManager == null) {
+                mitmManager = ImpersonatingMitmManager.builder()
+                        .rootCertificateSource(new KeyStoreFileCertificateSource(KEYSTORE_TYPE, KEYSTORE_RESOURCE, KEYSTORE_PRIVATE_KEY_ALIAS, KEYSTORE_PASSWORD))
+                        .serverKeyGenerator(new RSAKeyGenerator())
+                        .trustAllServers(trustAllServers)
+                        .build();
+            }
+
+            bootstrap.withManInTheMiddle(mitmManager);
         }
 
         if (readBandwidthLimitBps > 0 || writeBandwidthLimitBps > 0) {
@@ -868,7 +895,7 @@ public void setIdleConnectionTimeout(int idleConnectionTimeout, TimeUnit timeUni
     public void setRequestTimeout(int requestTimeout, TimeUnit timeUnit) {
         //TODO: implement Request Timeouts using LittleProxy. currently this only sets an idle connection timeout, if the idle connection
         // timeout is higher than the specified requestTimeout.
-        if (idleConnectionTimeoutSec == 0 || idleConnectionTimeoutSec >  TimeUnit.SECONDS.convert(requestTimeout, timeUnit)) {
+        if (idleConnectionTimeoutSec == 0 || idleConnectionTimeoutSec > TimeUnit.SECONDS.convert(requestTimeout, timeUnit)) {
             setIdleConnectionTimeout(requestTimeout, timeUnit);
         }
     }
@@ -1148,7 +1175,7 @@ public boolean waitForQuiescence(long quietPeriod, long timeout, TimeUnit timeUn
     /**
      * Instructs this proxy to route traffic through an upstream proxy. Proxy chaining is not compatible with man-in-the-middle
      * SSL, so HAR capture will be disabled for HTTPS traffic when using an upstream proxy.
-     * <p/>
+     * <p>
      * <b>Note:</b> Using {@link #setChainedProxyManager(ChainedProxyManager)} will supersede any value set by this method.
      *
      * @param chainedProxyAddress address of the upstream proxy
@@ -1181,7 +1208,6 @@ public void setChainedProxyManager(ChainedProxyManager chainedProxyManager) {
      * Configures the Netty thread pool used by the LittleProxy back-end. See {@link ThreadPoolConfiguration} for details.
      *
      * @param threadPoolConfiguration thread pool configuration to use
-     *
      */
     public void setThreadPoolConfiguration(ThreadPoolConfiguration threadPoolConfiguration) {
         if (isStarted()) {
@@ -1362,6 +1388,20 @@ public void setMitmDisabled(boolean mitmDisabled) throws IllegalStateException {
         this.mitmDisabled = mitmDisabled;
     }
 
+    @Override
+    public void setMitmManager(MitmManager mitmManager) {
+        this.mitmManager = mitmManager;
+    }
+
+    @Override
+    public void setTrustAllServers(boolean trustAllServers) {
+        if (isStarted()) {
+            throw new IllegalStateException("Cannot disable upstream server verification after the proxy has been started");
+        }
+
+        this.trustAllServers = trustAllServers;
+    }
+
     public boolean isMitmDisabled() {
         return this.mitmDisabled;
     }
@@ -1501,4 +1541,5 @@ public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerCont
             });
         }
     }
+
 }

browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/ModifiedRequestAwareFilter.java

@@ -1,12 +1,11 @@
 package net.lightbody.bmp.filters;
 
-import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpRequest;
 
 /**
  * Indicates that a filter wishes to capture the final HttpRequest that is sent to the server, reflecting all
- * modifications from request filters. {@link BrowserMobHttpFilterChain#clientToProxyRequest(HttpObject)} will invoke
- * the {@link #setModifiedHttpRequest(HttpRequest)} method <b>after</b> all filters have processed the initial
+ * modifications from request filters. {@link BrowserMobHttpFilterChain#clientToProxyRequest(io.netty.handler.codec.http.HttpObject)}
+ * will invoke the {@link #setModifiedHttpRequest(HttpRequest)} method <b>after</b> all filters have processed the initial
  * {@link HttpRequest} object.
  */
 public interface ModifiedRequestAwareFilter {

browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/filters/RewriteUrlFilterTest.groovy

@@ -156,6 +156,7 @@ class RewriteUrlFilterTest extends MockServerTest {
                 .withBody("success"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.rewriteUrl('https://localhost:(\\d+)/badresource', 'https://localhost:$1/rewrittenresource')
 
         proxy.start()

browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/BlacklistTest.groovy

@@ -56,6 +56,7 @@ class BlacklistTest extends MockServerTest {
                 .withBody("this URL should never be called"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
         int proxyPort = proxy.getPort()
 
@@ -128,6 +129,7 @@ class BlacklistTest extends MockServerTest {
                 .withBody("not blacklisted"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
         int proxyPort = proxy.getPort()
 

browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/NewHarTest.groovy

@@ -115,6 +115,7 @@ class NewHarTest extends MockServerTest {
 
         proxy = new BrowserMobProxyServer();
         proxy.setHarCaptureTypes([CaptureType.RESPONSE_COOKIES] as Set)
+        proxy.setTrustAllServers(true)
         proxy.start()
 
         proxy.newHar()
@@ -457,7 +458,8 @@ class NewHarTest extends MockServerTest {
                 .withStatusCode(200)
                 .withBody("success"))
 
-        proxy = new BrowserMobProxyServer();
+        proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
 
         proxy.newHar()
@@ -497,6 +499,7 @@ class NewHarTest extends MockServerTest {
 
         proxy = new BrowserMobProxyServer();
         proxy.rewriteUrl("https://localhost:${mockServerPort}/originalurl(.*)", "https://localhost:${mockServerPort}/httpsrewrittenurlcaptured\$1")
+        proxy.setTrustAllServers(true)
         proxy.start()
 
         proxy.newHar()
@@ -855,6 +858,7 @@ class NewHarTest extends MockServerTest {
                 .withBody("success"))
 
         proxy = new BrowserMobProxyServer();
+        proxy.setTrustAllServers(true)
         proxy.setIdleConnectionTimeout(3, TimeUnit.SECONDS)
         proxy.start()
 

browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/WhitelistTest.groovy

@@ -77,6 +77,7 @@ class WhitelistTest extends MockServerTest {
                 .withBody("should never be returned"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
         int proxyPort = proxy.getPort()
 
@@ -127,6 +128,7 @@ class WhitelistTest extends MockServerTest {
                 .withBody("whitelisted"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
         int proxyPort = proxy.getPort()
 
@@ -199,6 +201,7 @@ class WhitelistTest extends MockServerTest {
                 .withBody("should never be returned"))
 
         proxy = new BrowserMobProxyServer()
+        proxy.setTrustAllServers(true)
         proxy.start()
         int proxyPort = proxy.getPort()