From 91cfb4436b93199627c9ef019cd86b9ce11b51f6 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Mon, 18 Jan 2021 11:50:21 -0800 Subject: [PATCH 1/6] RBF: improve router connections management HDFS-15757 RBF: Improve Router connection management --- .../metrics/FederationRPCMBean.java | 15 +++++ .../metrics/FederationRPCMetrics.java | 10 +++ .../federation/router/ConnectionContext.java | 49 ++++++++++---- .../federation/router/ConnectionManager.java | 44 ++++++++++++- .../federation/router/ConnectionPool.java | 65 +++++++++++++++---- .../federation/router/RouterRpcClient.java | 18 +++++ 6 files changed, 173 insertions(+), 28 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java index 3cde5e5b93cab..a4469a3025a11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java @@ -76,6 +76,21 @@ public interface FederationRPCMBean { */ int getRpcClientNumActiveConnections(); + /** + * Get the number of idle RPC connections between the Router and the NNs. + * @return Number of idle RPC connections between the Router and the NNs. + */ + int getRpcClientNumIdleConnections(); + + /** + * Get the number of recently active RPC connections between + * the Router and the NNs. + * + * @return Number of recently active RPC connections between + * the Router and the NNs. + */ + int getRpcClientNumActiveConnectionsRecently(); + /** * Get the number of RPC connections to be created. * @return Number of RPC connections to be created. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index 887d50bf3d5d5..1e6aa8050d92e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -208,6 +208,16 @@ public int getRpcClientNumActiveConnections() { return rpcServer.getRPCClient().getNumActiveConnections(); } + @Override + public int getRpcClientNumIdleConnections() { + return rpcServer.getRPCClient().getNumIdleConnections(); + } + + @Override + public int getRpcClientNumActiveConnectionsRecently() { + return rpcServer.getRPCClient().getNumActiveConnectionsRecently(); + } + @Override public int getRpcClientNumCreatingConnections() { return rpcServer.getRPCClient().getNumCreatingConnections(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index 02a3dbeb4ea53..4a3811d65a3d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.util.Time; /** * Context to track a connection in a {@link ConnectionPool}. When a client uses @@ -42,7 +43,10 @@ public class ConnectionContext { private int numThreads = 0; /** If the connection is closed. */ private boolean closed = false; - + /** Last timestamp the connection was active. */ + private long lastActiveTs = 0; + /** The connection's active status would expire after this window. */ + private long activeWindow = 30 * 1000; public ConnectionContext(ProxyAndInfo connection) { this.client = connection; @@ -57,6 +61,17 @@ public synchronized boolean isActive() { return this.numThreads > 0; } + /** + * Check if the connection is/was active recently. + * + * @return True if the connection is active or + * was active in the past period of time. + */ + public synchronized boolean isActiveRecently() { + return isActive() || + Time.monotonicNow() - this.lastActiveTs <= activeWindow; + } + /** * Check if the connection is closed. * @@ -83,30 +98,38 @@ public synchronized boolean isUsable() { */ public synchronized ProxyAndInfo getClient() { this.numThreads++; + this.lastActiveTs = Time.monotonicNow(); return this.client; } /** - * Release this connection. If the connection was closed, close the proxy. - * Otherwise, mark the connection as not used by us anymore. + * Release this connection. */ public synchronized void release() { - if (--this.numThreads == 0 && this.closed) { - close(); + if (this.numThreads > 0) { + this.numThreads--; } } /** - * We will not use this connection anymore. If it's not being used, we close - * it. Otherwise, we let release() do it once we are done with it. + * Close a connection. Only idle connections can be closed since + * the RPC proxy would be shut down immediately. + * + * @param force whether the connection should be closed anyway. + * @throws IllegalStateException when the connection is not idle */ - public synchronized void close() { - this.closed = true; - if (this.numThreads == 0) { - Object proxy = this.client.getProxy(); - // Nobody should be using this anymore so it should close right away - RPC.stopProxy(proxy); + public synchronized void close(boolean force) throws IllegalStateException { + if (!force && this.numThreads > 0) { + throw new IllegalStateException("Active connection cannot be closed"); } + this.closed = true; + Object proxy = this.client.getProxy(); + // Nobody should be using this anymore so it should close right away + RPC.stopProxy(proxy); + } + + public synchronized void close() { + close(false); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 9ec3b54ed50b7..b2d1ffa446eb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -281,6 +281,42 @@ public int getNumActiveConnections() { return total; } + /** + * Get number of idle connections. + * + * @return Number of active connections. + */ + public int getNumIdleConnections() { + int total = 0; + readLock.lock(); + try { + for (ConnectionPool pool : this.pools.values()) { + total += pool.getNumIdleConnections(); + } + } finally { + readLock.unlock(); + } + return total; + } + + /** + * Get number of recently active connections. + * + * @return Number of recently active connections. + */ + public int getNumActiveConnectionsRecently() { + int total = 0; + readLock.lock(); + try { + for (ConnectionPool pool : this.pools.values()) { + total += pool.getNumActiveConnectionsRecently(); + } + } finally { + readLock.unlock(); + } + return total; + } + /** * Get the number of connections to be created. * @@ -327,12 +363,14 @@ void cleanup(ConnectionPool pool) { // Check if the pool hasn't been active in a while or not 50% are used long timeSinceLastActive = Time.now() - pool.getLastActiveTime(); int total = pool.getNumConnections(); - int active = pool.getNumActiveConnections(); + int active = pool.getNumActiveConnectionsRecently(); float poolMinActiveRatio = pool.getMinActiveRatio(); if (timeSinceLastActive > connectionCleanupPeriodMs || active < poolMinActiveRatio * total) { - // Remove and close 1 connection - List conns = pool.removeConnections(1); + // Be greedy here to close as many connections as possible in one shot + int targetConnectionsCount = (int)(poolMinActiveRatio * total) - active; + List conns = + pool.removeConnections(targetConnectionsCount); for (ConnectionContext conn : conns) { conn.close(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 52e7cebd26017..dfea12783258f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -252,19 +252,21 @@ public synchronized void addConnection(ConnectionContext conn) { */ public synchronized List removeConnections(int num) { List removed = new LinkedList<>(); - - // Remove and close the last connection - List tmpConnections = new ArrayList<>(); - for (int i=0; i this.minSize) { + int targetCount = Math.min(num, this.connections.size() - this.minSize); + // Remove and close targetCount of connections + List tmpConnections = new ArrayList<>(); + for (int i=0; i tmpConnections = this.connections; + for (ConnectionContext conn : tmpConnections) { + if (conn.isUsable()) { + ret++; + } + } + return ret; + } + + /** + * Number of active connections recently in the pool. + * + * @return Number of active connections recently. + */ + protected int getNumActiveConnectionsRecently() { + int ret = 0; + List tmpConnections = this.connections; + for (ConnectionContext conn : tmpConnections) { + if (conn.isActiveRecently()) { + ret++; + } + } + return ret; + } + /** * Get the last time the connection pool was used. * @@ -331,12 +366,18 @@ public String toString() { public String getJSON() { final Map info = new LinkedHashMap<>(); info.put("active", Integer.toString(getNumActiveConnections())); + info.put("recent_active", + Integer.toString(getNumActiveConnectionsRecently())); + info.put("idle", Integer.toString(getNumIdleConnections())); info.put("total", Integer.toString(getNumConnections())); if (LOG.isDebugEnabled()) { List tmpConnections = this.connections; for (int i=0; i Date: Mon, 25 Jan 2021 15:07:21 -0800 Subject: [PATCH 2/6] Fix error --- .../hadoop/hdfs/server/federation/router/ConnectionPool.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index dfea12783258f..2d51b9f07d3a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -261,6 +261,8 @@ public synchronized List removeConnections(int num) { // Only pick idle connections to close if (removed.size() < targetCount && conn.isUsable()) { removed.add(conn); + } else { + tmpConnections.add(conn); } } this.connections = tmpConnections; From 3e6880e001768be48c59f922188d65a0dfd9dca6 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Fri, 29 Jan 2021 16:45:49 -0800 Subject: [PATCH 3/6] Add tests --- .../federation/router/ConnectionContext.java | 3 +- .../federation/router/ConnectionManager.java | 9 +++++- .../router/TestConnectionManager.java | 28 ++++++++++++------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index 4a3811d65a3d9..4fcd1d7d030e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.federation.router; import java.net.InetSocketAddress; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.ipc.RPC; @@ -46,7 +47,7 @@ public class ConnectionContext { /** Last timestamp the connection was active. */ private long lastActiveTs = 0; /** The connection's active status would expire after this window. */ - private long activeWindow = 30 * 1000; + private long activeWindow = TimeUnit.SECONDS.toMillis(30); public ConnectionContext(ProxyAndInfo connection) { this.client = connection; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index b2d1ffa446eb0..2b08fc818eb24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -363,12 +363,19 @@ void cleanup(ConnectionPool pool) { // Check if the pool hasn't been active in a while or not 50% are used long timeSinceLastActive = Time.now() - pool.getLastActiveTime(); int total = pool.getNumConnections(); + // Active is a transient status in many cases for a connection since + // the handler thread uses the connection very quickly. Thus the number + // of connections with handlers using at the call time is constantly low. + // Recently active is more lasting status and it shows how many + // connections have been used with a recent time period. (i.e. 30 seconds) int active = pool.getNumActiveConnectionsRecently(); float poolMinActiveRatio = pool.getMinActiveRatio(); if (timeSinceLastActive > connectionCleanupPeriodMs || active < poolMinActiveRatio * total) { // Be greedy here to close as many connections as possible in one shot - int targetConnectionsCount = (int)(poolMinActiveRatio * total) - active; + // The number should at least be 1 + int targetConnectionsCount = Math.max(1, + (int)(poolMinActiveRatio * total) - active); List conns = pool.removeConnections(targetConnectionsCount); for (ConnectionContext conn : conns) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java index dffd3d8d12375..e397692e9a86d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java @@ -255,6 +255,9 @@ private void checkPoolConnections(UserGroupInformation ugi, if (e.getKey().getUgi() == ugi) { assertEquals(numOfConns, e.getValue().getNumConnections()); assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections()); + // idle + active = total connections + assertEquals(numOfConns - numOfActiveConns, + e.getValue().getNumIdleConnections()); connPoolFoundForUser = true; } } @@ -265,13 +268,19 @@ private void checkPoolConnections(UserGroupInformation ugi, @Test public void testConfigureConnectionActiveRatio() throws IOException { - final int totalConns = 10; - int activeConns = 7; + // test 1 conn below the threshold and these conns are closed + testConnectionCleanup(0.8f, 10, 7, 9); + + // test 2 conn below the threshold and these conns are closed + testConnectionCleanup(0.8f, 10, 6, 8); + } + private void testConnectionCleanup(float ratio, int totalConns, + int activeConns, int leftConns) throws IOException { Configuration tmpConf = new Configuration(); - // Set dfs.federation.router.connection.min-active-ratio 0.8f + // Set dfs.federation.router.connection.min-active-ratio tmpConf.setFloat( - RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_MIN_ACTIVE_RATIO, 0.8f); + RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_MIN_ACTIVE_RATIO, ratio); ConnectionManager tmpConnManager = new ConnectionManager(tmpConf); tmpConnManager.start(); @@ -284,21 +293,20 @@ public void testConfigureConnectionActiveRatio() throws IOException { TEST_NN_ADDRESS, NamenodeProtocol.class); ConnectionPool pool = poolMap.get(connectionPoolId); - // Test min active ratio is 0.8f - assertEquals(0.8f, pool.getMinActiveRatio(), 0.001f); + // Test min active ratio is as set value + assertEquals(ratio, pool.getMinActiveRatio(), 0.001f); pool.getConnection().getClient(); // Test there is one active connection in pool assertEquals(1, pool.getNumActiveConnections()); - // Add other 6 active/9 total connections to pool + // Add other active-1 connections / totalConns-1 connections to pool addConnectionsToPool(pool, totalConns - 1, activeConns - 1); - // There are 7 active connections. - // The active number is less than totalConns(10) * minActiveRatio(0.8f). + // There are activeConn connections. // We can cleanup the pool tmpConnManager.cleanup(pool); - assertEquals(totalConns - 1, pool.getNumConnections()); + assertEquals(leftConns, pool.getNumConnections()); tmpConnManager.close(); } From 9f5489055688e28d14c7bd1a9afa129fd38d8708 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Mon, 1 Feb 2021 20:05:06 -0800 Subject: [PATCH 4/6] Use recently active connections when determining creation --- .../hadoop/hdfs/server/federation/router/ConnectionManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 2b08fc818eb24..b773a79948edc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -459,7 +459,7 @@ public void run() { ConnectionPool pool = this.queue.take(); try { int total = pool.getNumConnections(); - int active = pool.getNumActiveConnections(); + int active = pool.getNumActiveConnectionsRecently(); float poolMinActiveRatio = pool.getMinActiveRatio(); if (pool.getNumConnections() < pool.getMaxSize() && active >= poolMinActiveRatio * total) { From 28afba3eaf18e984e80327a94cc002c4afdada65 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Tue, 2 Feb 2021 11:22:08 -0800 Subject: [PATCH 5/6] Address comments --- .../hdfs/server/federation/router/ConnectionContext.java | 5 ++--- .../hadoop/hdfs/server/federation/router/ConnectionPool.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index 4fcd1d7d030e7..619a1bee97227 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -47,7 +47,7 @@ public class ConnectionContext { /** Last timestamp the connection was active. */ private long lastActiveTs = 0; /** The connection's active status would expire after this window. */ - private long activeWindow = TimeUnit.SECONDS.toMillis(30); + private final static long ACTIVE_WINDOW_TIME = TimeUnit.SECONDS.toMillis(30); public ConnectionContext(ProxyAndInfo connection) { this.client = connection; @@ -69,8 +69,7 @@ public synchronized boolean isActive() { * was active in the past period of time. */ public synchronized boolean isActiveRecently() { - return isActive() || - Time.monotonicNow() - this.lastActiveTs <= activeWindow; + return Time.monotonicNow() - this.lastActiveTs <= ACTIVE_WINDOW_TIME; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 2d51b9f07d3a8..827e62ce3ef1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -256,7 +256,7 @@ public synchronized List removeConnections(int num) { int targetCount = Math.min(num, this.connections.size() - this.minSize); // Remove and close targetCount of connections List tmpConnections = new ArrayList<>(); - for (int i=0; i Date: Tue, 18 May 2021 22:39:55 -0700 Subject: [PATCH 6/6] Comments --- .../federation/router/ConnectionContext.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index 619a1bee97227..9a5434b91ce2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Context to track a connection in a {@link ConnectionPool}. When a client uses @@ -38,6 +40,9 @@ */ public class ConnectionContext { + private static final Logger LOG = + LoggerFactory.getLogger(ConnectionContext.class); + /** Client for the connection. */ private final ProxyAndInfo client; /** How many threads are using this connection. */ @@ -116,11 +121,14 @@ public synchronized void release() { * the RPC proxy would be shut down immediately. * * @param force whether the connection should be closed anyway. - * @throws IllegalStateException when the connection is not idle */ - public synchronized void close(boolean force) throws IllegalStateException { + public synchronized void close(boolean force) { if (!force && this.numThreads > 0) { - throw new IllegalStateException("Active connection cannot be closed"); + // this is an erroneous case but we have to close the connection + // anyway since there will be connection leak if we don't do so + // the connection has been moved out of the pool + LOG.error("Active connection with {} handlers will be closed", + this.numThreads); } this.closed = true; Object proxy = this.client.getProxy();