kernel: atomics: support for 64-bit atomic operations

doc/reference/kernel/other/atomic.rst

@@ -3,8 +3,9 @@
 Atomic Services
 ###############
 
-An :dfn:`atomic variable` is a 32-bit variable that can be read and modified
-by threads and ISRs in an uninterruptible manner.
+An :dfn:`atomic variable` is one that can be read and modified
+by threads and ISRs in an uninterruptible manner. It 32-bit on
+32-bit machines and 64-bit on 64-bit machines.
 
 .. contents::
     :local:

drivers/ieee802154/ieee802154_kw41z.c

@@ -706,7 +706,7 @@ static int kw41z_tx(const struct device *dev, enum ieee802154_tx_mode mode,
 		handle_ack(kw41z, frag->data[2]);
 	}
 
-	LOG_DBG("seq_retval: %d", kw41z->seq_retval);
+	LOG_DBG("seq_retval: %ld", kw41z->seq_retval);
 	return kw41z->seq_retval;
 }
 

drivers/serial/uart_nrfx_uart.c

@@ -79,7 +79,8 @@ static struct {
 
 	bool tx_abort;
 	const uint8_t *volatile tx_buffer;
-	size_t tx_buffer_length;
+	/* note: this is aliased with atomic_t in uart_nrfx_poll_out() */
+	unsigned long tx_buffer_length;
 	volatile size_t tx_counter;
 #if HW_FLOW_CONTROL_AVAILABLE
 	int32_t tx_timeout;

include/sys/atomic.h

@@ -13,12 +13,13 @@
 #include <stddef.h>
 
 #include <zephyr/types.h>
+#include <sys/util_macro.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef int atomic_t;
+typedef long atomic_t;
 typedef atomic_t atomic_val_t;
 typedef void *atomic_ptr_t;
 typedef atomic_ptr_t atomic_ptr_val_t;
@@ -76,7 +77,7 @@ typedef atomic_ptr_t atomic_ptr_val_t;
  */
 
 #define ATOMIC_BITS (sizeof(atomic_val_t) * 8)
-#define ATOMIC_MASK(bit) (1U << ((uint32_t)(bit) & (ATOMIC_BITS - 1U)))
+#define ATOMIC_MASK(bit) BIT((unsigned long)(bit) & (ATOMIC_BITS - 1U))
 #define ATOMIC_ELEM(addr, bit) ((addr) + ((bit) / ATOMIC_BITS))
 
 /**

subsys/bluetooth/audio/otc.c

@@ -1645,5 +1645,5 @@ static void l2cap_status(struct bt_l2cap_chan *chan, atomic_t *status)
 {
 	int credits = atomic_get(&l2ch_chan.ch.tx.credits);
 
-	BT_DBG("Channel %p status %u, credits %d", chan, *status, credits);
+	BT_DBG("Channel %p status %lu, credits %d", chan, *status, credits);
 }

subsys/bluetooth/host/att.c

@@ -436,7 +436,7 @@ static inline bool att_chan_is_connected(struct bt_att_chan *chan)
 static int bt_att_chan_send(struct bt_att_chan *chan, struct net_buf *buf,
 			    bt_att_chan_sent_t cb)
 {
-	BT_DBG("chan %p flags %u code 0x%02x", chan, atomic_get(chan->flags),
+	BT_DBG("chan %p flags %lu code 0x%02x", chan, atomic_get(chan->flags),
 	       ((struct bt_att_hdr *)buf->data)->code);
 
 	return chan_send(chan, buf, cb);
@@ -2610,7 +2610,7 @@ static struct bt_att_chan *att_get_fixed_chan(struct bt_conn *conn)
 
 static void att_chan_attach(struct bt_att *att, struct bt_att_chan *chan)
 {
-	BT_DBG("att %p chan %p flags %u", att, chan, atomic_get(chan->flags));
+	BT_DBG("att %p chan %p flags %lu", att, chan, atomic_get(chan->flags));
 
 	if (sys_slist_is_empty(&att->chans)) {
 		/* Init general queues when attaching the first channel */

subsys/bluetooth/host/conn.c

@@ -1078,7 +1078,7 @@ struct bt_conn *bt_conn_ref(struct bt_conn *conn)
 		}
 	} while (!atomic_cas(&conn->ref, old, old + 1));
 
-	BT_DBG("handle %u ref %d -> %d", conn->handle, old, old + 1);
+	BT_DBG("handle %u ref %ld -> %ld", conn->handle, old, old + 1);
 
 	return conn;
 }
@@ -1089,7 +1089,7 @@ void bt_conn_unref(struct bt_conn *conn)
 
 	old = atomic_dec(&conn->ref);
 
-	BT_DBG("handle %u ref %d -> %d", conn->handle, old,
+	BT_DBG("handle %u ref %ld -> %ld", conn->handle, old,
 	       atomic_get(&conn->ref));
 
 	__ASSERT(old > 0, "Conn reference counter is 0");

subsys/bluetooth/host/l2cap.c

@@ -1855,7 +1855,7 @@ static int l2cap_chan_le_send(struct bt_l2cap_le_chan *ch,
 		return -EAGAIN;
 	}
 
-	BT_DBG("ch %p cid 0x%04x len %u credits %u", ch, ch->tx.cid,
+	BT_DBG("ch %p cid 0x%04x len %u credits %lu", ch, ch->tx.cid,
 	       seg->len, atomic_get(&ch->tx.credits));
 
 	len = seg->len - sdu_hdr_len;
@@ -1998,7 +1998,7 @@ static void le_credits(struct bt_l2cap *l2cap, uint8_t ident,
 
 	l2cap_chan_tx_give_credits(ch, credits);
 
-	BT_DBG("chan %p total credits %u", ch, atomic_get(&ch->tx.credits));
+	BT_DBG("chan %p total credits %lu", ch, atomic_get(&ch->tx.credits));
 
 	l2cap_chan_tx_resume(ch);
 }
@@ -2172,7 +2172,7 @@ static void l2cap_chan_send_credits(struct bt_l2cap_le_chan *chan,
 
 	l2cap_send(chan->chan.conn, BT_L2CAP_CID_LE_SIG, buf);
 
-	BT_DBG("chan %p credits %u", chan, atomic_get(&chan->rx.credits));
+	BT_DBG("chan %p credits %lu", chan, atomic_get(&chan->rx.credits));
 }
 
 static void l2cap_chan_update_credits(struct bt_l2cap_le_chan *chan,

subsys/bluetooth/services/ots/ots_l2cap.c

@@ -122,7 +122,7 @@ static int l2cap_recv(struct bt_l2cap_chan *chan, struct net_buf *buf)
 
 static void l2cap_status(struct bt_l2cap_chan *chan, atomic_t *status)
 {
-	LOG_DBG("Channel %p status %u", chan, *status);
+	LOG_DBG("Channel %p status %lu", chan, atomic_get(status));
 }
 
 static void l2cap_connected(struct bt_l2cap_chan *chan)

subsys/net/ip/net_pkt.c

@@ -538,7 +538,7 @@ void net_pkt_unref(struct net_pkt *pkt)
 
 #if NET_LOG_LEVEL >= LOG_LEVEL_DBG
 #if CONFIG_NET_PKT_LOG_LEVEL >= LOG_LEVEL_DBG
-	NET_DBG("%s [%d] pkt %p ref %d frags %p (%s():%d)",
+	NET_DBG("%s [%d] pkt %p ref %ld frags %p (%s():%d)",
 		slab2str(pkt->slab), k_mem_slab_num_free_get(pkt->slab),
 		pkt, ref - 1, pkt->frags, caller, line);
 #endif
@@ -619,7 +619,7 @@ struct net_pkt *net_pkt_ref(struct net_pkt *pkt)
 	} while (!atomic_cas(&pkt->atomic_ref, ref, ref + 1));
 
 #if CONFIG_NET_PKT_LOG_LEVEL >= LOG_LEVEL_DBG
-	NET_DBG("%s [%d] pkt %p ref %d (%s():%d)",
+	NET_DBG("%s [%d] pkt %p ref %ld (%s():%d)",
 		slab2str(pkt->slab), k_mem_slab_num_free_get(pkt->slab),
 		pkt, ref + 1, caller, line);
 #endif

subsys/net/ip/net_private.h

@@ -248,14 +248,14 @@ static inline void net_pkt_print_buffer_info(struct net_pkt *pkt, const char *st
 		printk("%s", str);
 	}
 
-	printk("%p[%d]", pkt, atomic_get(&pkt->atomic_ref));
+	printk("%p[%ld]", pkt, atomic_get(&pkt->atomic_ref));
 
 	if (buf) {
 		printk("->");
 	}
 
 	while (buf) {
-		printk("%p[%d/%u (%u/%u)]", buf, atomic_get(&pkt->atomic_ref),
+		printk("%p[%ld/%u (%u/%u)]", buf, atomic_get(&pkt->atomic_ref),
 		       buf->len, net_buf_max_len(buf), buf->size);
 
 		buf = buf->frags;

subsys/net/ip/net_shell.c

@@ -1496,7 +1496,7 @@ static void tcp_sent_list_cb(struct tcp *conn, void *user_data)
 			details->printed_details = true;
 		}
 
-		PR("%p   %d    %u\t %u\t  %zd\t  %d\t  %d/%d/%d %s\n",
+		PR("%p   %ld    %u\t %u\t  %zd\t  %d\t  %d/%d/%d %s\n",
 		   conn, atomic_get(&conn->ref_count), conn->recv_win,
 		   conn->send_win, conn->send_data_total, conn->unacked_len,
 		   conn->in_retransmission, conn->in_connect, conn->in_close,
@@ -1524,12 +1524,12 @@ static void tcp_sent_list_cb(struct tcp *conn, void *user_data)
 			struct net_buf *frag = pkt->frags;
 
 			if (!details->printed_send_queue_header) {
-				PR("%p[%d/%zd]", pkt,
+				PR("%p[%ld/%zd]", pkt,
 				   atomic_get(&pkt->atomic_ref),
 				   net_pkt_get_len(pkt));
 				details->printed_send_queue_header = true;
 			} else {
-				PR("                %p[%d/%zd]",
+				PR("                %p[%ld/%zd]",
 				   pkt, atomic_get(&pkt->atomic_ref),
 				   net_pkt_get_len(pkt));
 			}
@@ -1630,7 +1630,7 @@ static void allocs_cb(struct net_pkt *pkt,
 
 	if (func_alloc) {
 		if (in_use) {
-			PR("%p/%d\t%5s\t%5s\t%s():%d\n",
+			PR("%p/%ld\t%5s\t%5s\t%s():%d\n",
 			   pkt, atomic_get(&pkt->atomic_ref), str,
 			   net_pkt_slab2str(pkt->slab),
 			   func_alloc, line_alloc);
@@ -3632,8 +3632,7 @@ static void context_info(struct net_context *context, void *user_data)
 		}
 
 #if defined(CONFIG_NET_BUF_POOL_USAGE)
-		PR("%p\t%d\t%d\tEDATA (%s)\n",
-		   pool, pool->buf_count,
+		PR("%p\t%d\t%ld\tEDATA (%s)\n", pool, pool->buf_count,
 		   atomic_get(&pool->avail_count), pool->name);
 #else
 		PR("%p\t%d\tEDATA\n", pool, pool->buf_count);
@@ -3671,13 +3670,11 @@ static int cmd_net_mem(const struct shell *shell, size_t argc, char *argv[])
 	PR("%p\t%d\t%u\tTX\n",
 	       tx, tx->num_blocks, k_mem_slab_num_free_get(tx));
 
-	PR("%p\t%d\t%d\tRX DATA (%s)\n",
-	       rx_data, rx_data->buf_count,
-	       atomic_get(&rx_data->avail_count), rx_data->name);
+	PR("%p\t%d\t%ld\tRX DATA (%s)\n	", rx_data, rx_data->buf_count,
+	   atomic_get(&rx_data->avail_count), rx_data->name);
 
-	PR("%p\t%d\t%d\tTX DATA (%s)\n",
-	       tx_data, tx_data->buf_count,
-	       atomic_get(&tx_data->avail_count), tx_data->name);
+	PR("%p\t%d\t%ld\tTX DATA (%s)\n", tx_data, tx_data->buf_count,
+	   atomic_get(&tx_data->avail_count), tx_data->name);
 #else
 	PR("Address\t\tTotal\tName\n");
 
@@ -4241,14 +4238,14 @@ static void net_pkt_buffer_info(const struct shell *shell, struct net_pkt *pkt)
 	struct net_buf *buf = pkt->buffer;
 
 	PR("net_pkt %p buffer chain:\n", pkt);
-	PR("%p[%d]", pkt, atomic_get(&pkt->atomic_ref));
+	PR("%p[%ld]", pkt, atomic_get(&pkt->atomic_ref));
 
 	if (buf) {
 		PR("->");
 	}
 
 	while (buf) {
-		PR("%p[%d/%u (%u/%u)]", buf, atomic_get(&pkt->atomic_ref),
+		PR("%p[%ld/%u (%u/%u)]", buf, atomic_get(&pkt->atomic_ref),
 		   buf->len, net_buf_max_len(buf), buf->size);
 
 		buf = buf->frags;

subsys/net/l2/ethernet/arp.c

@@ -37,7 +37,7 @@ static void arp_entry_cleanup(struct arp_entry *entry, bool pending)
 	NET_DBG("%p", entry);
 
 	if (pending) {
-		NET_DBG("Releasing pending pkt %p (ref %d)",
+		NET_DBG("Releasing pending pkt %p (ref %ld)",
 			entry->pending,
 			atomic_get(&entry->pending->atomic_ref) - 1);
 		net_pkt_unref(entry->pending);

subsys/portability/cmsis_rtos_v2/thread.c

@@ -26,8 +26,8 @@ static const osThreadAttr_t init_thread_attrs = {
 
 static sys_dlist_t thread_list;
 static struct cv2_thread cv2_thread_pool[CONFIG_CMSIS_V2_THREAD_MAX_COUNT];
-static uint32_t thread_num;
-static uint32_t thread_num_dynamic;
+static atomic_t thread_num;
+static atomic_t thread_num_dynamic;
 
 #if CONFIG_CMSIS_V2_THREAD_DYNAMIC_MAX_COUNT != 0
 static K_THREAD_STACK_ARRAY_DEFINE(cv2_thread_stack_pool,		     \

subsys/shell/shell_cmds.c

@@ -310,7 +310,7 @@ static int cmd_shell_stats_show(const struct shell *shell, size_t argc,
 	ARG_UNUSED(argc);
 	ARG_UNUSED(argv);
 
-	shell_print(shell, "Lost logs: %u", shell->stats->log_lost_cnt);
+	shell_print(shell, "Lost logs: %lu", shell->stats->log_lost_cnt);
 
 	return 0;
 }

subsys/usb/class/netusb/function_rndis.c

@@ -457,7 +457,7 @@ static void rndis_notify_rsp(void)
 	};
 	int ret;
 
-	LOG_DBG("count %u", atomic_get(&rndis.notify_count));
+	LOG_DBG("count %lu", atomic_get(&rndis.notify_count));
 
 	if (atomic_get(&rndis.notify_count)) {
 		LOG_WRN("Notification is already sent");

tests/kernel/common/src/atomic.c

@@ -7,6 +7,10 @@
 #include <ztest.h>
 #include <sys/atomic.h>
 
+/* convenience macro - return either 64-bit or 32-bit value */
+#define ATOMIC_WORD(val_if_64, val_if_32)                                                          \
+	((atomic_t)((sizeof(void *) == sizeof(uint64_t)) ? (val_if_64) : (val_if_32)))
+
 /* an example of the number of atomic bit in an array */
 #define NUM_FLAG_BITS 100
 
@@ -102,6 +106,9 @@ void test_atomic(void)
 
 	ATOMIC_DEFINE(flag_bits, NUM_FLAG_BITS) = {0};
 
+	zassert_equal(sizeof(atomic_t), ATOMIC_WORD(sizeof(uint64_t), sizeof(uint32_t)),
+		      "sizeof(atomic_t)");
+
 	target = 4;
 	value = 5;
 	oldvalue = 6;
@@ -211,63 +218,63 @@ void test_atomic(void)
 	target = 0xFF00;
 	value  = 0x0F0F;
 	zassert_true((atomic_nand(&target, value) == 0xFF00), "atomic_nand");
-	zassert_true((target == 0xFFFFF0FF), "atomic_nand");
+	zassert_true((target == ATOMIC_WORD(0xFFFFFFFFFFFFF0FF, 0xFFFFF0FF)), "atomic_nand");
 
 	/* atomic_test_bit() */
-	for (i = 0; i < 32; i++) {
-		target = 0x0F0F0F0F;
-		zassert_true(!!(atomic_test_bit(&target, i) == !!(target & (1 << i))),
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		target = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
+		zassert_true(!!(atomic_test_bit(&target, i) == !!(target & BIT(i))),
 			     "atomic_test_bit");
 	}
 
 	/* atomic_test_and_clear_bit() */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
-		zassert_true(!!(atomic_test_and_clear_bit(&target, i)) == !!(orig & (1 << i)),
+		zassert_true(!!(atomic_test_and_clear_bit(&target, i)) == !!(orig & BIT(i)),
 			     "atomic_test_and_clear_bit");
-		zassert_true(target == (orig & ~(1 << i)), "atomic_test_and_clear_bit");
+		zassert_true(target == (orig & ~BIT(i)), "atomic_test_and_clear_bit");
 	}
 
 	/* atomic_test_and_set_bit() */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
-		zassert_true(!!(atomic_test_and_set_bit(&target, i)) == !!(orig & (1 << i)),
+		zassert_true(!!(atomic_test_and_set_bit(&target, i)) == !!(orig & BIT(i)),
 			     "atomic_test_and_set_bit");
-		zassert_true(target == (orig | (1 << i)), "atomic_test_and_set_bit");
+		zassert_true(target == (orig | BIT(i)), "atomic_test_and_set_bit");
 	}
 
 	/* atomic_clear_bit() */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
 		atomic_clear_bit(&target, i);
-		zassert_true(target == (orig & ~(1 << i)), "atomic_clear_bit");
+		zassert_true(target == (orig & ~BIT(i)), "atomic_clear_bit");
 	}
 
 	/* atomic_set_bit() */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
 		atomic_set_bit(&target, i);
-		zassert_true(target == (orig | (1 << i)), "atomic_set_bit");
+		zassert_true(target == (orig | BIT(i)), "atomic_set_bit");
 	}
 
 	/* atomic_set_bit_to(&target, i, false) */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
 		atomic_set_bit_to(&target, i, false);
-		zassert_true(target == (orig & ~(1 << i)), "atomic_set_bit_to");
+		zassert_true(target == (orig & ~BIT(i)), "atomic_set_bit_to");
 	}
 
 	/* atomic_set_bit_to(&target, i, true) */
-	for (i = 0; i < 32; i++) {
-		orig = 0x0F0F0F0F;
+	for (i = 0; i < ATOMIC_BITS; i++) {
+		orig = ATOMIC_WORD(0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F);
 		target = orig;
 		atomic_set_bit_to(&target, i, true);
-		zassert_true(target == (orig | (1 << i)), "atomic_set_bit_to");
+		zassert_true(target == (orig | BIT(i)), "atomic_set_bit_to");
 	}
 
 	/* ATOMIC_DEFINE */