mm: zswap: properly synchronize freeing resources during CPU hotunplug

JIRA: https://issues.redhat.com/browse/RHEL-78678
CVE: CVE-2025-21693
Conflicts:
  * the order and context of several hunks in this backport does not
    match their upstream commit of reference because RHEL-9 misses the
    full series (and dependencies) that introduces commit fa9ad6e
    ("mm: zswap: break out zwap_compress()"). These churny changesets,
    however, are mostly cosmetical and none of them are required for
    the purpose of this backport.

commit 12dcb0e
Author: Yosry Ahmed <[email protected]>
Date:   Wed Jan 8 22:24:41 2025 +0000

    mm: zswap: properly synchronize freeing resources during CPU hotunplug

    In zswap_compress() and zswap_decompress(), the per-CPU acomp_ctx of the
    current CPU at the beginning of the operation is retrieved and used
    throughout.  However, since neither preemption nor migration are disabled,
    it is possible that the operation continues on a different CPU.

    If the original CPU is hotunplugged while the acomp_ctx is still in use,
    we run into a UAF bug as some of the resources attached to the acomp_ctx
    are freed during hotunplug in zswap_cpu_comp_dead() (i.e.
    acomp_ctx.buffer, acomp_ctx.req, or acomp_ctx.acomp).

    The problem was introduced in commit 1ec3b5f ("mm/zswap: move to use
    crypto_acomp API for hardware acceleration") when the switch to the
    crypto_acomp API was made.  Prior to that, the per-CPU crypto_comp was
    retrieved using get_cpu_ptr() which disables preemption and makes sure the
    CPU cannot go away from under us.  Preemption cannot be disabled with the
    crypto_acomp API as a sleepable context is needed.

    Use the acomp_ctx.mutex to synchronize CPU hotplug callbacks allocating
    and freeing resources with compression/decompression paths.  Make sure
    that acomp_ctx.req is NULL when the resources are freed.  In the
    compression/decompression paths, check if acomp_ctx.req is NULL after
    acquiring the mutex (meaning the CPU was offlined) and retry on the new
    CPU.

    The initialization of acomp_ctx.mutex is moved from the CPU hotplug
    callback to the pool initialization where it belongs (where the mutex is
    allocated).  In addition to adding clarity, this makes sure that CPU
    hotplug cannot reinitialize a mutex that is already locked by
    compression/decompression.

    Previously a fix was attempted by holding cpus_read_lock() [1].  This
    would have caused a potential deadlock as it is possible for code already
    holding the lock to fall into reclaim and enter zswap (causing a
    deadlock).  A fix was also attempted using SRCU for synchronization, but
    Johannes pointed out that synchronize_srcu() cannot be used in CPU hotplug
    notifiers [2].

    Alternative fixes that were considered/attempted and could have worked:
    - Refcounting the per-CPU acomp_ctx. This involves complexity in
      handling the race between the refcount dropping to zero in
      zswap_[de]compress() and the refcount being re-initialized when the
      CPU is onlined.
    - Disabling migration before getting the per-CPU acomp_ctx [3], but
      that's discouraged and is a much bigger hammer than needed, and could
      result in subtle performance issues.

    [1]https://lkml.kernel.org/[email protected]/
    [2]https://lkml.kernel.org/[email protected]/
    [3]https://lkml.kernel.org/[email protected]/

    [[email protected]: remove comment]
      Link: https://lkml.kernel.org/r/CAJD7tkaxS1wjn+swugt8QCvQ-rVF5RZnjxwPGX17k8x9zSManA@mail.gmail.com
    Link: https://lkml.kernel.org/r/[email protected]
    Fixes: 1ec3b5f ("mm/zswap: move to use crypto_acomp API for hardware acceleration")
    Signed-off-by: Yosry Ahmed <[email protected]>
    Reported-by: Johannes Weiner <[email protected]>
    Closes: https://lore.kernel.org/lkml/[email protected]/
    Reported-by: Sam Sun <[email protected]>
    Closes: https://lore.kernel.org/lkml/CAEkJfYMtSdM5HceNsXUDf5haghD5+o2e7Qv4OcuruL4tPg6OaQ@mail.gmail.com/
    Cc: Barry Song <[email protected]>
    Cc: Chengming Zhou <[email protected]>
    Cc: Kanchana P Sridhar <[email protected]>
    Cc: Nhat Pham <[email protected]>
    Cc: Vitaly Wool <[email protected]>
    Cc: <[email protected]>
    Signed-off-by: Andrew Morton <[email protected]>

Signed-off-by: Rafael Aquini <[email protected]>

Author: Rafael Aquini

mm/zswap.c

@@ -714,11 +714,12 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 	struct acomp_req *req;
 	int ret;
 
-	mutex_init(&acomp_ctx->mutex);
-
+	mutex_lock(&acomp_ctx->mutex);
 	acomp_ctx->buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
-	if (!acomp_ctx->buffer)
-		return -ENOMEM;
+	if (!acomp_ctx->buffer) {
+		ret = -ENOMEM;
+		goto buffer_fail;
+	}
 
 	acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
 	if (IS_ERR(acomp)) {
@@ -747,12 +748,15 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 	acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 				   crypto_req_done, &acomp_ctx->wait);
 
+	mutex_unlock(&acomp_ctx->mutex);
 	return 0;
 
 req_fail:
 	crypto_free_acomp(acomp_ctx->acomp);
 acomp_fail:
 	kfree(acomp_ctx->buffer);
+buffer_fail:
+	mutex_unlock(&acomp_ctx->mutex);
 	return ret;
 }
 
@@ -761,17 +765,45 @@ static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
 
+	mutex_lock(&acomp_ctx->mutex);
 	if (!IS_ERR_OR_NULL(acomp_ctx)) {
 		if (!IS_ERR_OR_NULL(acomp_ctx->req))
 			acomp_request_free(acomp_ctx->req);
+		acomp_ctx->req = NULL;
 		if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
 			crypto_free_acomp(acomp_ctx->acomp);
 		kfree(acomp_ctx->buffer);
 	}
+	mutex_unlock(&acomp_ctx->mutex);
 
 	return 0;
 }
 
+static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool)
+{
+	struct crypto_acomp_ctx *acomp_ctx;
+
+	for (;;) {
+		acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
+		mutex_lock(&acomp_ctx->mutex);
+		if (likely(acomp_ctx->req))
+			return acomp_ctx;
+		/*
+		 * It is possible that we were migrated to a different CPU after
+		 * getting the per-CPU ctx but before the mutex was acquired. If
+		 * the old CPU got offlined, zswap_cpu_comp_dead() could have
+		 * already freed ctx->req (among other things) and set it to
+		 * NULL. Just try again on the new CPU that we ended up on.
+		 */
+		mutex_unlock(&acomp_ctx->mutex);
+	}
+}
+
+static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx)
+{
+	mutex_unlock(&acomp_ctx->mutex);
+}
+
 /*********************************
 * pool functions
 **********************************/
@@ -1029,7 +1061,7 @@ static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
 	struct zswap_pool *pool;
 	char name[38]; /* 'zswap' + 32 char (max) num + \0 */
 	gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
-	int ret;
+	int ret, cpu;
 
 	if (!zswap_has_pool) {
 		/* if either are unset, pool initialization failed, and we
@@ -1067,6 +1099,9 @@ static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
 		goto error;
 	}
 
+	for_each_possible_cpu(cpu)
+		mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex);
+
 	ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
 				       &pool->node);
 	if (ret)
@@ -1376,9 +1411,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	struct crypto_acomp_ctx *acomp_ctx;
 	u8 *src;
 
-	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
-	mutex_lock(&acomp_ctx->mutex);
-
+	acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
 	if (!zpool_can_sleep_mapped(zpool)) {
 		memcpy(acomp_ctx->buffer, src, entry->length);
@@ -1392,10 +1425,10 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
 	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
 	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
-	mutex_unlock(&acomp_ctx->mutex);
 
 	if (zpool_can_sleep_mapped(zpool))
 		zpool_unmap_handle(zpool, entry->handle);
+	acomp_ctx_put_unlock(acomp_ctx);
 }
 
 /*********************************
@@ -1612,9 +1645,7 @@ bool zswap_store(struct folio *folio)
 	}
 
 	/* compress */
-	acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
-
-	mutex_lock(&acomp_ctx->mutex);
+	acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
 
 	dst = acomp_ctx->buffer;
 	sg_init_table(&input, 1);
@@ -1662,7 +1693,7 @@ bool zswap_store(struct folio *folio)
 	buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
 	memcpy(buf, dst, dlen);
 	zpool_unmap_handle(zpool, handle);
-	mutex_unlock(&acomp_ctx->mutex);
+	acomp_ctx_put_unlock(acomp_ctx);
 
 	/* populate entry */
 	entry->swpentry = swp_entry(type, offset);
@@ -1705,7 +1736,7 @@ bool zswap_store(struct folio *folio)
 	return true;
 
 put_dstmem:
-	mutex_unlock(&acomp_ctx->mutex);
+	acomp_ctx_put_unlock(acomp_ctx);
 put_pool:
 	zswap_pool_put(entry->pool);
 freepage: