init: add sub-priority to internal ordering

include/zephyr/device.h

@@ -848,6 +848,17 @@ static inline bool z_impl_device_is_ready(const struct device *dev)
 
 #endif /* CONFIG_DEVICE_DEPS */
 
+/**
+ * @brief Init sub-priority of the device
+ *
+ * The sub-priority is defined by the devicetree ordinal, which ensures that
+ * multiple drivers running at the same priority level run in an order that
+ * respects the devicetree dependencies.
+ */
+#define Z_DEVICE_INIT_SUB_PRIO(node_id)                                        \
+	COND_CODE_1(DT_NODE_EXISTS(node_id),                                   \
+		    (DT_DEP_ORD_STR_SORTABLE(node_id)), (0))
+
 /**
  * @brief Maximum device name length.
  *
@@ -923,14 +934,17 @@ static inline bool z_impl_device_is_ready(const struct device *dev)
 /**
  * @brief Define the init entry for a device.
  *
+ * @param node_id Devicetree node id for the device (DT_INVALID_NODE if a
+ * software device).
  * @param dev_id Device identifier.
  * @param init_fn_ Device init function.
  * @param level Initialization level.
  * @param prio Initialization priority.
  */
-#define Z_DEVICE_INIT_ENTRY_DEFINE(dev_id, init_fn_, level, prio)              \
-	static const Z_DECL_ALIGN(struct init_entry)                           \
-		Z_INIT_ENTRY_SECTION(level, prio) __used __noasan              \
+#define Z_DEVICE_INIT_ENTRY_DEFINE(node_id, dev_id, init_fn_, level, prio)     \
+	static const Z_DECL_ALIGN(struct init_entry) __used __noasan           \
+		Z_INIT_ENTRY_SECTION(level, prio,                              \
+				     Z_DEVICE_INIT_SUB_PRIO(node_id))          \
 		Z_INIT_ENTRY_NAME(DEVICE_NAME_GET(dev_id)) = {                 \
 			.init_fn = {.dev = (init_fn_)},                        \
 			.dev = &DEVICE_NAME_GET(dev_id),                       \
@@ -967,7 +981,7 @@ static inline bool z_impl_device_is_ready(const struct device *dev)
 	Z_DEVICE_BASE_DEFINE(node_id, dev_id, name, pm, data, config, level,   \
 			     prio, api, state, Z_DEVICE_DEPS_NAME(dev_id));    \
                                                                                \
-	Z_DEVICE_INIT_ENTRY_DEFINE(dev_id, init_fn, level, prio)
+	Z_DEVICE_INIT_ENTRY_DEFINE(node_id, dev_id, init_fn, level, prio)
 
 #if defined(CONFIG_HAS_DTS) || defined(__DOXYGEN__)
 /**

include/zephyr/devicetree/ordinals.h

@@ -24,6 +24,13 @@
  */
 #define DT_DEP_ORD(node_id) DT_CAT(node_id, _ORD)
 
+/**
+ * @brief Get a node's dependency ordinal in string sortable form
+ * @param node_id Node identifier
+ * @return the node's dependency ordinal as a zero-padded integer literal
+ */
+#define DT_DEP_ORD_STR_SORTABLE(node_id) DT_CAT(node_id, _ORD_STR_SORTABLE)
+
 /**
  * @brief Get a list of dependency ordinals of a node's direct dependencies
  *

include/zephyr/init.h

@@ -128,10 +128,12 @@ struct init_entry {
  * @brief Init entry section.
  *
  * Each init entry is placed in a section with a name crafted so that it allows
- * linker scripts to sort them according to the specified level/priority.
+ * linker scripts to sort them according to the specified
+ * level/priority/sub-priority.
  */
-#define Z_INIT_ENTRY_SECTION(level, prio)                                      \
-	__attribute__((__section__(".z_init_" #level STRINGIFY(prio)"_")))
+#define Z_INIT_ENTRY_SECTION(level, prio, sub_prio)                           \
+	__attribute__((__section__(                                           \
+		".z_init_" #level STRINGIFY(prio)"_" STRINGIFY(sub_prio)"_")))
 
 /** @endcond */
 
@@ -186,7 +188,7 @@ struct init_entry {
  */
 #define SYS_INIT_NAMED(name, init_fn_, level, prio)                            \
 	static const Z_DECL_ALIGN(struct init_entry)                           \
-		Z_INIT_ENTRY_SECTION(level, prio) __used __noasan              \
+		Z_INIT_ENTRY_SECTION(level, prio, 0) __used __noasan           \
 		Z_INIT_ENTRY_NAME(name) = {                                    \
 			.init_fn = {.sys = (init_fn_)},                        \
 			.dev = NULL,                                           \

scripts/build/check_init_priorities.py

@@ -68,20 +68,23 @@ class Priority:
     def __init__(self, name):
         for idx, level in enumerate(_DEVICE_INIT_LEVELS):
             if level in name:
-                _, priority = name.strip("_").split(level)
+                _, priority_str = name.strip("_").split(level)
+                priority, sub_priority = priority_str.split("_")
                 self._level = idx
                 self._priority = int(priority)
-                self._level_priority = self._level * 100 + self._priority
+                self._sub_priority = int(sub_priority)
+                # Tuples compare elementwise in order
+                self._level_priority = (self._level, self._priority, self._sub_priority)
                 return
 
         raise ValueError("Unknown level in %s" % name)
 
     def __repr__(self):
-        return "<%s %s %d>" % (self.__class__.__name__,
-                               _DEVICE_INIT_LEVELS[self._level], self._priority)
+        return "<%s %s %d %d>" % (self.__class__.__name__,
+                               _DEVICE_INIT_LEVELS[self._level], self._priority, self._sub_priority)
 
     def __str__(self):
-        return "%s %d" % (_DEVICE_INIT_LEVELS[self._level], self._priority)
+        return "%s %d %d" % (_DEVICE_INIT_LEVELS[self._level], self._priority, self._sub_priority)
 
     def __lt__(self, other):
         return self._level_priority < other._level_priority

scripts/build/check_init_priorities_test.py

@@ -19,17 +19,17 @@ class TestPriority(unittest.TestCase):
     """Tests for the Priority class."""
 
     def test_priority_parsing(self):
-        prio1 = check_init_priorities.Priority(".rel.z_init_POST_KERNEL12_")
-        self.assertEqual(prio1._level_priority, 312)
+        prio1 = check_init_priorities.Priority(".rel.z_init_POST_KERNEL12_0_")
+        self.assertEqual(prio1._level_priority, (3, 12, 0))
 
-        prio2 = check_init_priorities.Priority("noisenoise_POST_KERNEL99_")
-        self.assertEqual(prio2._level_priority, 399)
+        prio2 = check_init_priorities.Priority("noisenoise_POST_KERNEL99_00023_")
+        self.assertEqual(prio2._level_priority, (3, 99, 23))
 
-        prio3 = check_init_priorities.Priority("_PRE_KERNEL_10_")
-        self.assertEqual(prio3._level_priority, 100)
+        prio3 = check_init_priorities.Priority("_PRE_KERNEL_10_99999_")
+        self.assertEqual(prio3._level_priority, (1, 0, 99999))
 
-        prio4 = check_init_priorities.Priority("_PRE_KERNEL_110_")
-        self.assertEqual(prio4._level_priority, 110)
+        prio4 = check_init_priorities.Priority("_PRE_KERNEL_110_00001_")
+        self.assertEqual(prio4._level_priority, (1, 10, 1))
 
         with self.assertRaises(ValueError):
             check_init_priorities.Priority("i-am-not-a-priority")
@@ -40,32 +40,35 @@ def test_priority_parsing(self):
 
     def test_priority_levels(self):
         prios = [
-                check_init_priorities.Priority(".rel.z_init_EARLY0_"),
-                check_init_priorities.Priority(".rel.z_init_EARLY1_"),
-                check_init_priorities.Priority(".rel.z_init_EARLY11_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_10_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_11_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_111_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_20_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_21_"),
-                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_211_"),
-                check_init_priorities.Priority(".rel.z_init_POST_KERNEL0_"),
-                check_init_priorities.Priority(".rel.z_init_POST_KERNEL1_"),
-                check_init_priorities.Priority(".rel.z_init_POST_KERNEL11_"),
-                check_init_priorities.Priority(".rel.z_init_APPLICATION0_"),
-                check_init_priorities.Priority(".rel.z_init_APPLICATION1_"),
-                check_init_priorities.Priority(".rel.z_init_APPLICATION11_"),
-                check_init_priorities.Priority(".rel.z_init_SMP0_"),
-                check_init_priorities.Priority(".rel.z_init_SMP1_"),
-                check_init_priorities.Priority(".rel.z_init_SMP11_"),
+                check_init_priorities.Priority(".rel.z_init_EARLY0_0_"),
+                check_init_priorities.Priority(".rel.z_init_EARLY1_0_"),
+                check_init_priorities.Priority(".rel.z_init_EARLY11_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_10_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_11_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_111_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_111_1_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_111_00002_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_111_00010_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_20_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_21_0_"),
+                check_init_priorities.Priority(".rel.z_init_PRE_KERNEL_211_0_"),
+                check_init_priorities.Priority(".rel.z_init_POST_KERNEL0_0_"),
+                check_init_priorities.Priority(".rel.z_init_POST_KERNEL1_0_"),
+                check_init_priorities.Priority(".rel.z_init_POST_KERNEL11_0_"),
+                check_init_priorities.Priority(".rel.z_init_APPLICATION0_0_"),
+                check_init_priorities.Priority(".rel.z_init_APPLICATION1_0_"),
+                check_init_priorities.Priority(".rel.z_init_APPLICATION11_0_"),
+                check_init_priorities.Priority(".rel.z_init_SMP0_0_"),
+                check_init_priorities.Priority(".rel.z_init_SMP1_0_"),
+                check_init_priorities.Priority(".rel.z_init_SMP11_0_"),
                 ]
 
         self.assertListEqual(prios, sorted(prios))
 
     def test_priority_strings(self):
-        prio = check_init_priorities.Priority(".rel.z_init_POST_KERNEL12_")
-        self.assertEqual(str(prio), "POST_KERNEL 12")
-        self.assertEqual(repr(prio), "<Priority POST_KERNEL 12>")
+        prio = check_init_priorities.Priority(".rel.z_init_POST_KERNEL12_00023_")
+        self.assertEqual(str(prio), "POST_KERNEL 12 23")
+        self.assertEqual(repr(prio), "<Priority POST_KERNEL 12 23>")
 
 class testZephyrObjectFile(unittest.TestCase):
     """Tests for the ZephyrObjectFile class."""

scripts/dts/gen_defines.py

@@ -747,6 +747,7 @@ def fmt_dep_list(dep_list):
 
     out_comment("Node's dependency ordinal:")
     out_dt_define(f"{node.z_path_id}_ORD", node.dep_ordinal)
+    out_dt_define(f"{node.z_path_id}_ORD_STR_SORTABLE", f"{node.dep_ordinal:0>5}")
 
     out_comment("Ordinals for what this node depends on directly:")
     out_dt_define(f"{node.z_path_id}_REQUIRES_ORDS",

tests/kernel/device/app.overlay

@@ -51,4 +51,21 @@
 			    "dale";
 		status = "okay";
 	};
+
+	fakedomain_0: fakedomain_0 {
+		compatible = "fakedomain";
+		status = "okay";
+		power-domain = <&fakedomain_2>;
+	};
+
+	fakedomain_1: fakedomain_1 {
+		compatible = "fakedomain";
+		status = "okay";
+		power-domain = <&fakedomain_0>;
+	};
+
+	fakedomain_2: fakedomain_2 {
+		compatible = "fakedomain";
+		status = "okay";
+	};
 };

tests/kernel/device/boards/hifive_unmatched.overlay

@@ -48,4 +48,21 @@
 			    "dale";
 		status = "okay";
 	};
+
+	fakedomain_0: fakedomain_0 {
+		compatible = "fakedomain";
+		status = "okay";
+		power-domain = <&fakedomain_2>;
+	};
+
+	fakedomain_1: fakedomain_1 {
+		compatible = "fakedomain";
+		status = "okay";
+		power-domain = <&fakedomain_0>;
+	};
+
+	fakedomain_2: fakedomain_2 {
+		compatible = "fakedomain";
+		status = "okay";
+	};
 };

tests/kernel/device/dts/bindings/fakedomain.yml

@@ -0,0 +1,8 @@
+# Copyright (c) 2023, CSIRO
+# SPDX-License-Identifier: Apache-2.0
+
+description: Properties for fake power domain
+
+compatible: "fakedomain"
+
+include: base.yaml

tests/kernel/device/src/main.c

@@ -265,6 +265,7 @@ ZTEST(device, test_sys_init_multiple)
 /* this is for storing sequence during initialization */
 extern int init_level_sequence[4];
 extern int init_priority_sequence[4];
+extern int init_sub_priority_sequence[3];
 extern unsigned int seq_level_cnt;
 extern unsigned int seq_priority_cnt;
 
@@ -298,7 +299,7 @@ ZTEST(device, test_device_init_level)
 /**
  * @brief Test initialization priorities for device driver instances
  *
- * details After the defined device instances have initialized, we check the
+ * @details After the defined device instances have initialized, we check the
  * sequence number that each driver stored during initialization. If the
  * sequence of initial priority stored is corresponding with our expectation, it
  * means assigning the priority for driver instance works.
@@ -322,6 +323,25 @@ ZTEST(device, test_device_init_priority)
 			"init sequence is not correct");
 }
 
+/**
+ * @brief Test initialization sub-priorities for device driver instances
+ *
+ * @details After the defined device instances have initialized, we check the
+ * sequence number that each driver stored during initialization. If the
+ * sequence of initial priority stored is corresponding with our expectation, it
+ * means using the devicetree for sub-priority sorting works.
+ *
+ * @ingroup kernel_device_tests
+ */
+ZTEST(device, test_device_init_sub_priority)
+{
+	/* fakedomain_1 depends on fakedomain_0 which depends on fakedomain_2,
+	 * therefore we require that the initialisation runs in the reverse order.
+	 */
+	zassert_equal(init_sub_priority_sequence[0], 1, "");
+	zassert_equal(init_sub_priority_sequence[1], 2, "");
+	zassert_equal(init_sub_priority_sequence[2], 0, "");
+}
 
 /**
  * @brief Test abstraction of device drivers with common functionalities

tests/kernel/device/src/test_driver_init.c

@@ -39,8 +39,10 @@
 /* this is for storing sequence during initialization */
 __pinned_bss int init_level_sequence[4] = {0};
 __pinned_bss int init_priority_sequence[4] = {0};
+__pinned_bss int init_sub_priority_sequence[3] = {0};
 __pinned_bss unsigned int seq_level_cnt;
 __pinned_bss unsigned int seq_priority_cnt;
+__pinned_bss unsigned int seq_sub_priority_cnt;
 
 /* define driver type 1: for testing initialize levels and priorities */
 typedef int (*my_api_configure_t)(const struct device *dev, int dev_config);
@@ -126,6 +128,28 @@ static int my_driver_pri_4_init(const struct device *dev)
 	return 0;
 }
 
+/* driver init function of testing sub_priority */
+static int my_driver_sub_pri_0_init(const struct device *dev)
+{
+	init_sub_priority_sequence[0] = seq_sub_priority_cnt++;
+
+	return 0;
+}
+
+static int my_driver_sub_pri_1_init(const struct device *dev)
+{
+	init_sub_priority_sequence[1] = seq_sub_priority_cnt++;
+
+	return 0;
+}
+
+static int my_driver_sub_pri_2_init(const struct device *dev)
+{
+	init_sub_priority_sequence[2] = seq_sub_priority_cnt++;
+
+	return 0;
+}
+
 /**
  * @brief Test providing control device driver initialization order
  *
@@ -172,3 +196,13 @@ DEVICE_DEFINE(my_driver_priority_2, MY_DRIVER_PRI_2,
 DEVICE_DEFINE(my_driver_priority_3, MY_DRIVER_PRI_3,
 		&my_driver_pri_3_init, NULL, NULL, NULL, POST_KERNEL, 3,
 		&funcs_my_drivers);
+
+/* Create several devices at the same init priority that depend on each
+ * other in devicetree so that we can validate linker sorting.
+ */
+DEVICE_DT_DEFINE(DT_NODELABEL(fakedomain_0), my_driver_sub_pri_0_init,
+		NULL, NULL, NULL, APPLICATION, 33, NULL);
+DEVICE_DT_DEFINE(DT_NODELABEL(fakedomain_1), my_driver_sub_pri_1_init,
+		NULL, NULL, NULL, APPLICATION, 33, NULL);
+DEVICE_DT_DEFINE(DT_NODELABEL(fakedomain_2), my_driver_sub_pri_2_init,
+		NULL, NULL, NULL, APPLICATION, 33, NULL);

tests/lib/devicetree/api/src/main.c

@@ -25,6 +25,8 @@
 #include <zephyr/drivers/gpio.h>
 #include <zephyr/drivers/mbox.h>
 
+#include <stdlib.h>
+
 #define TEST_CHILDREN	DT_PATH(test, test_children)
 #define TEST_DEADBEEF	DT_PATH(test, gpio_deadbeef)
 #define TEST_ABCD1234	DT_PATH(test, gpio_abcd1234)
@@ -2440,6 +2442,28 @@ ZTEST(devicetree_api, test_dep_ord)
 	}
 }
 
+ZTEST(devicetree_api, test_dep_ord_str_sortable)
+{
+	const char *root_ord = STRINGIFY(DT_DEP_ORD_STR_SORTABLE(DT_ROOT));
+
+	/* Root ordinal is always 0 */
+	zassert_mem_equal(root_ord, "00000", 6);
+
+	/* Test string sortable versions equal decimal values.
+	 * We go through the STRINGIFY()->atoi conversion cycle to avoid
+	 * the C compiler treating the number as octal due to leading zeros.
+	 * atoi() simply ignores them.
+	 */
+	zassert_equal(atoi(STRINGIFY(DT_DEP_ORD_STR_SORTABLE(DT_ROOT))),
+		      DT_DEP_ORD(DT_ROOT), "Invalid sortable string");
+	zassert_equal(atoi(STRINGIFY(DT_DEP_ORD_STR_SORTABLE(TEST_DEADBEEF))),
+		      DT_DEP_ORD(TEST_DEADBEEF), "Invalid sortable string");
+	zassert_equal(atoi(STRINGIFY(DT_DEP_ORD_STR_SORTABLE(TEST_TEMP))),
+		      DT_DEP_ORD(TEST_TEMP), "Invalid sortable string");
+	zassert_equal(atoi(STRINGIFY(DT_DEP_ORD_STR_SORTABLE(TEST_REG))),
+		      DT_DEP_ORD(TEST_REG), "Invalid sortable string");
+}
+
 ZTEST(devicetree_api, test_path)
 {
 	zassert_true(!strcmp(DT_NODE_PATH(DT_ROOT), "/"), "");

tests/misc/check_init_priorities/prj.conf

@@ -1 +1,2 @@
-# Empty
+# Required to force 2 stage linking
+CONFIG_DEVICE_DEPS=y