net/rds: Add krefs to struct rds_connection

We need to add krefs to struct rds_connection to make
connection destroy/reaping safer and this encompasses
the following code changes:

rds_conn_create() does a kref_init() when a new connection
is created. Transports (rds_rdma, rds_tcp, loopback) will now
call rds_conn_destroy_init() as the destroy conn mechanism is
split into two phases of init (quiesce) and fini (free).
In the first destroy_init/quiesce phase all worker threads
are cancelled and pending messages are dropped/torn-down.
The second free phase begins when the conn->c_refcount
drops to 0 and rds_conn_destroy_fini() is called by the
kref callback.

kref coverage via rds_conn_get()/rds_conn_put()
includes the following class of functions:
- rds_conn_lookup(), rds_conn_laddr_list(),
  rds_conn_drop_sock_cancel_worker(), rds_conn_ha_changed_task(),
  rds_send_worker(), rds_recv_worker(), rds_hb_worker().
- af_rds: rds(6)_user_reset(), rds_sock->rs_conn/rds_sock_put().
- congestion: rds_cong_{add,remove}_conn().
- send & recv: rds_sendmsg(), qp & cm_id contexts.
- All callers of rds_conn_create().

Orabug: 38118655

Reviewed-by: Gerd Rausch <[email protected]>
Signed-off-by: Sharath Srinivasan <[email protected]>
Signed-off-by: Vijayendra Suman <[email protected]>

Author: sharathmsrini

net/rds/af_rds.c

@@ -98,7 +98,7 @@ static struct rt_debug_tp rt_debug_tp_map[] = {
 	{ RDS_RTD_ERR_EXT,
 	  { "rds_send_worker_err", "rds_receive_worker_err", NULL }},
 	{ RDS_RTD_CM,
-	  {  "rds_conn_destroy", "rds_conn_drop",
+	  { "rds_conn_destroy_init", "rds_conn_destroy_fini", "rds_conn_drop",
 	    "rds_ib_cm_initiate_connect_err", "rds_ib_conn_path_connect",
 	    "rds_rdma_cm_event_handler", "rds_rdma_cm_event_handler_err",
 	    NULL }},
@@ -584,8 +584,10 @@ static int rds_user_reset(struct rds_sock *rs, sockptr_t optval, int optlen)
 		}
 
 		list_for_each_entry(conn, &s_addr_conns, c_laddr_node)
-			if (conn)
+			if (conn) {
 				rds_user_conn_paths_drop(conn);
+				rds_conn_put(conn); /* for rds_conn_laddr_list */
+			}
 		mutex_unlock(&conn_reset_zero_dest);
 		goto done;
 	}
@@ -603,6 +605,7 @@ static int rds_user_reset(struct rds_sock *rs, sockptr_t optval, int optlen)
 		       &reset.src.s_addr,
 		       &reset.dst.s_addr, conn->c_tos);
 		rds_user_conn_paths_drop(conn);
+		rds_conn_put(conn); /* for rds_conn_find */
 	}
 done:
 	return 0;
@@ -636,8 +639,10 @@ static int rds6_user_reset(struct rds_sock *rs, sockptr_t optval, int optlen)
 		}
 
 		list_for_each_entry(conn, &s_addr_conns, c_laddr_node)
-			if (conn)
+			if (conn) {
 				rds_user_conn_paths_drop(conn);
+				rds_conn_put(conn); /* for rds_conn_laddr_list */
+			}
 		mutex_unlock(&conn_reset_zero_dest);
 		goto done;
 	}
@@ -653,6 +658,7 @@ static int rds6_user_reset(struct rds_sock *rs, sockptr_t optval, int optlen)
 		       is_tcp ? "tcp" : "IB",
 		       &reset.src, &reset.dst, conn->c_tos);
 		rds_user_conn_paths_drop(conn);
+		rds_conn_put(conn); /* for rds_conn_find */
 	}
 done:
 	return 0;
@@ -1165,6 +1171,9 @@ void debug_sock_put(struct sock *sk)
 	if (refcount_dec_and_test(&sk->sk_refcnt)) {
 		struct rds_sock *rs = rds_sk_to_rs(sk);
 
+		if (rs->rs_conn)
+			rds_conn_put(rs->rs_conn); /* rs_conn from rds_sendmsg */
+
 		if (READ_ONCE(rs->poison) != RED_ACTIVE) {
 			pr_err_ratelimited("bad poison on put %llx\n", READ_ONCE(rs->poison));
 			WARN_ON(1);

net/rds/cong.c

@@ -289,6 +289,7 @@ void rds_cong_add_conn(struct rds_connection *conn)
 	unsigned long flags;
 
 	rdsdebug("conn %p now on map %p\n", conn, conn->c_lcong);
+	rds_conn_get(conn); /* put in rds_cong_remove_conn */
 	spin_lock_irqsave(&rds_cong_lock, flags);
 	list_add_tail(&conn->c_map_item, &conn->c_lcong->m_conn_list);
 	spin_unlock_irqrestore(&rds_cong_lock, flags);
@@ -302,6 +303,7 @@ void rds_cong_remove_conn(struct rds_connection *conn)
 	spin_lock_irqsave(&rds_cong_lock, flags);
 	list_del_init(&conn->c_map_item);
 	spin_unlock_irqrestore(&rds_cong_lock, flags);
+	rds_conn_put(conn); /* get in rds_cong_add_conn */
 }
 
 int rds_cong_get_maps(struct rds_connection *conn)