Skip to content

Commit 0e439ba

Browse files
nipung87gregkh
authored andcommitted
cdx: add MSI support for CDX bus
Add CDX-MSI domain per CDX controller with gic-its domain as a parent, to support MSI for CDX devices. CDX devices allocate MSIs from the CDX domain. Also, introduce APIs to alloc and free IRQs for CDX domain. In CDX subsystem firmware is a controller for all devices and their configuration. CDX bus controller sends all the write_msi_msg commands to firmware running on RPU and the firmware interfaces with actual devices to pass this information to devices Since, CDX controller is the only way to communicate with the Firmware for MSI write info, CDX domain per controller required in contrast to having a CDX domain per device. Co-developed-by: Nikhil Agarwal <[email protected]> Signed-off-by: Nikhil Agarwal <[email protected]> Co-developed-by: Abhijit Gangurde <[email protected]> Signed-off-by: Abhijit Gangurde <[email protected]> Signed-off-by: Nipun Gupta <[email protected]> Reviewed-by: Pieter Jansen van Vuuren <[email protected]> Reviewed-by: Thomas Gleixner <[email protected]> Tested-by: Nikhil Agarwal <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Greg Kroah-Hartman <[email protected]>
1 parent e3a5905 commit 0e439ba

File tree

10 files changed

+434
-3
lines changed

10 files changed

+434
-3
lines changed

drivers/cdx/Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,7 @@
88
ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CDX_BUS
99

1010
obj-$(CONFIG_CDX_BUS) += cdx.o controller/
11+
12+
ifdef CONFIG_GENERIC_MSI_IRQ
13+
obj-$(CONFIG_CDX_BUS) += cdx_msi.o
14+
endif

drivers/cdx/cdx.c

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
*/
5757

5858
#include <linux/init.h>
59+
#include <linux/irqdomain.h>
5960
#include <linux/kernel.h>
6061
#include <linux/of.h>
6162
#include <linux/of_device.h>
@@ -302,8 +303,19 @@ static int cdx_probe(struct device *dev)
302303
{
303304
struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
304305
struct cdx_device *cdx_dev = to_cdx_device(dev);
306+
struct cdx_controller *cdx = cdx_dev->cdx;
305307
int error;
306308

309+
/*
310+
* Setup MSI device data so that generic MSI alloc/free can
311+
* be used by the device driver.
312+
*/
313+
if (cdx->msi_domain) {
314+
error = msi_setup_device_data(&cdx_dev->dev);
315+
if (error)
316+
return error;
317+
}
318+
307319
error = cdx_drv->probe(cdx_dev);
308320
if (error) {
309321
dev_err_probe(dev, error, "%s failed\n", __func__);
@@ -787,6 +799,7 @@ int cdx_device_add(struct cdx_dev_params *dev_params)
787799

788800
/* Populate CDX dev params */
789801
cdx_dev->req_id = dev_params->req_id;
802+
cdx_dev->msi_dev_id = dev_params->msi_dev_id;
790803
cdx_dev->vendor = dev_params->vendor;
791804
cdx_dev->device = dev_params->device;
792805
cdx_dev->subsystem_vendor = dev_params->subsys_vendor;
@@ -804,12 +817,19 @@ int cdx_device_add(struct cdx_dev_params *dev_params)
804817
cdx_dev->dev.bus = &cdx_bus_type;
805818
cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
806819
cdx_dev->dev.release = cdx_device_release;
820+
cdx_dev->msi_write_pending = false;
821+
mutex_init(&cdx_dev->irqchip_lock);
807822

808823
/* Set Name */
809824
dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
810825
((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
811826
cdx_dev->dev_num);
812827

828+
if (cdx->msi_domain) {
829+
cdx_dev->num_msi = dev_params->num_msi;
830+
dev_set_msi_domain(&cdx_dev->dev, cdx->msi_domain);
831+
}
832+
813833
ret = device_add(&cdx_dev->dev);
814834
if (ret) {
815835
dev_err(&cdx_dev->dev,

drivers/cdx/cdx.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
* @req_id: Requestor ID associated with CDX device
2626
* @class: Class of the CDX Device
2727
* @revision: Revision of the CDX device
28+
* @msi_dev_id: MSI device ID associated with CDX device
29+
* @num_msi: Number of MSI's supported by the device
2830
*/
2931
struct cdx_dev_params {
3032
struct cdx_controller *cdx;
@@ -40,6 +42,8 @@ struct cdx_dev_params {
4042
u32 req_id;
4143
u32 class;
4244
u8 revision;
45+
u32 msi_dev_id;
46+
u32 num_msi;
4347
};
4448

4549
/**
@@ -79,4 +83,12 @@ int cdx_device_add(struct cdx_dev_params *dev_params);
7983
*/
8084
struct device *cdx_bus_add(struct cdx_controller *cdx, u8 bus_num);
8185

86+
/**
87+
* cdx_msi_domain_init - Init the CDX bus MSI domain.
88+
* @dev: Device of the CDX bus controller
89+
*
90+
* Return: CDX MSI domain, NULL on failure
91+
*/
92+
struct irq_domain *cdx_msi_domain_init(struct device *dev);
93+
8294
#endif /* _CDX_H_ */

drivers/cdx/cdx_msi.c

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/*
3+
* AMD CDX bus driver MSI support
4+
*
5+
* Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
6+
*/
7+
8+
#include <linux/of.h>
9+
#include <linux/of_device.h>
10+
#include <linux/of_address.h>
11+
#include <linux/of_irq.h>
12+
#include <linux/irq.h>
13+
#include <linux/irqdomain.h>
14+
#include <linux/msi.h>
15+
#include <linux/cdx/cdx_bus.h>
16+
17+
#include "cdx.h"
18+
19+
static void cdx_msi_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
20+
{
21+
struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
22+
struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev);
23+
24+
/* We would not operate on msg here rather we wait for irq_bus_sync_unlock()
25+
* to be called from preemptible task context.
26+
*/
27+
msi_desc->msg = *msg;
28+
cdx_dev->msi_write_pending = true;
29+
}
30+
31+
static void cdx_msi_write_irq_lock(struct irq_data *irq_data)
32+
{
33+
struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
34+
struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev);
35+
36+
mutex_lock(&cdx_dev->irqchip_lock);
37+
}
38+
39+
static void cdx_msi_write_irq_unlock(struct irq_data *irq_data)
40+
{
41+
struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
42+
struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev);
43+
struct cdx_controller *cdx = cdx_dev->cdx;
44+
struct cdx_device_config dev_config;
45+
46+
if (!cdx_dev->msi_write_pending) {
47+
mutex_unlock(&cdx_dev->irqchip_lock);
48+
return;
49+
}
50+
51+
cdx_dev->msi_write_pending = false;
52+
mutex_unlock(&cdx_dev->irqchip_lock);
53+
54+
dev_config.msi.msi_index = msi_desc->msi_index;
55+
dev_config.msi.data = msi_desc->msg.data;
56+
dev_config.msi.addr = ((u64)(msi_desc->msg.address_hi) << 32) | msi_desc->msg.address_lo;
57+
58+
/*
59+
* dev_configure() is a controller callback which can interact with
60+
* Firmware or other entities, and can sleep, so invoke this function
61+
* outside of the mutex held region.
62+
*/
63+
dev_config.type = CDX_DEV_MSI_CONF;
64+
if (cdx->ops->dev_configure)
65+
cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num, &dev_config);
66+
}
67+
68+
int cdx_enable_msi(struct cdx_device *cdx_dev)
69+
{
70+
struct cdx_controller *cdx = cdx_dev->cdx;
71+
struct cdx_device_config dev_config;
72+
73+
dev_config.type = CDX_DEV_MSI_ENABLE;
74+
dev_config.msi_enable = true;
75+
if (cdx->ops->dev_configure) {
76+
return cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num,
77+
&dev_config);
78+
}
79+
80+
return -EOPNOTSUPP;
81+
}
82+
EXPORT_SYMBOL_GPL(cdx_enable_msi);
83+
84+
void cdx_disable_msi(struct cdx_device *cdx_dev)
85+
{
86+
struct cdx_controller *cdx = cdx_dev->cdx;
87+
struct cdx_device_config dev_config;
88+
89+
dev_config.type = CDX_DEV_MSI_ENABLE;
90+
dev_config.msi_enable = false;
91+
if (cdx->ops->dev_configure)
92+
cdx->ops->dev_configure(cdx, cdx_dev->bus_num, cdx_dev->dev_num, &dev_config);
93+
}
94+
EXPORT_SYMBOL_GPL(cdx_disable_msi);
95+
96+
static struct irq_chip cdx_msi_irq_chip = {
97+
.name = "CDX-MSI",
98+
.irq_mask = irq_chip_mask_parent,
99+
.irq_unmask = irq_chip_unmask_parent,
100+
.irq_eoi = irq_chip_eoi_parent,
101+
.irq_set_affinity = msi_domain_set_affinity,
102+
.irq_write_msi_msg = cdx_msi_write_msg,
103+
.irq_bus_lock = cdx_msi_write_irq_lock,
104+
.irq_bus_sync_unlock = cdx_msi_write_irq_unlock
105+
};
106+
107+
/* Convert an msi_desc to a unique identifier within the domain. */
108+
static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev,
109+
struct msi_desc *desc)
110+
{
111+
return ((irq_hw_number_t)dev->msi_dev_id << 10) | desc->msi_index;
112+
}
113+
114+
static void cdx_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
115+
{
116+
arg->desc = desc;
117+
arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc);
118+
}
119+
120+
static int cdx_msi_prepare(struct irq_domain *msi_domain,
121+
struct device *dev,
122+
int nvec, msi_alloc_info_t *info)
123+
{
124+
struct cdx_device *cdx_dev = to_cdx_device(dev);
125+
struct device *parent = cdx_dev->cdx->dev;
126+
struct msi_domain_info *msi_info;
127+
u32 dev_id;
128+
int ret;
129+
130+
/* Retrieve device ID from requestor ID using parent device */
131+
ret = of_map_id(parent->of_node, cdx_dev->msi_dev_id, "msi-map", "msi-map-mask",
132+
NULL, &dev_id);
133+
if (ret) {
134+
dev_err(dev, "of_map_id failed for MSI: %d\n", ret);
135+
return ret;
136+
}
137+
138+
#ifdef GENERIC_MSI_DOMAIN_OPS
139+
/* Set the device Id to be passed to the GIC-ITS */
140+
info->scratchpad[0].ul = dev_id;
141+
#endif
142+
143+
msi_info = msi_get_domain_info(msi_domain->parent);
144+
145+
return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
146+
}
147+
148+
static struct msi_domain_ops cdx_msi_ops = {
149+
.msi_prepare = cdx_msi_prepare,
150+
.set_desc = cdx_msi_set_desc
151+
};
152+
153+
static struct msi_domain_info cdx_msi_domain_info = {
154+
.ops = &cdx_msi_ops,
155+
.chip = &cdx_msi_irq_chip,
156+
.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
157+
MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS
158+
};
159+
160+
struct irq_domain *cdx_msi_domain_init(struct device *dev)
161+
{
162+
struct device_node *np = dev->of_node;
163+
struct fwnode_handle *fwnode_handle;
164+
struct irq_domain *cdx_msi_domain;
165+
struct device_node *parent_node;
166+
struct irq_domain *parent;
167+
168+
fwnode_handle = of_node_to_fwnode(np);
169+
170+
parent_node = of_parse_phandle(np, "msi-map", 1);
171+
if (!parent_node) {
172+
dev_err(dev, "msi-map not present on cdx controller\n");
173+
return NULL;
174+
}
175+
176+
parent = irq_find_matching_fwnode(of_node_to_fwnode(parent_node), DOMAIN_BUS_NEXUS);
177+
if (!parent || !msi_get_domain_info(parent)) {
178+
dev_err(dev, "unable to locate ITS domain\n");
179+
return NULL;
180+
}
181+
182+
cdx_msi_domain = msi_create_irq_domain(fwnode_handle, &cdx_msi_domain_info, parent);
183+
if (!cdx_msi_domain) {
184+
dev_err(dev, "unable to create CDX-MSI domain\n");
185+
return NULL;
186+
}
187+
188+
dev_dbg(dev, "CDX-MSI domain created\n");
189+
190+
return cdx_msi_domain;
191+
}
192+
EXPORT_SYMBOL_NS_GPL(cdx_msi_domain_init, CDX_BUS_CONTROLLER);

drivers/cdx/controller/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ if CDX_BUS
99

1010
config CDX_CONTROLLER
1111
tristate "CDX bus controller"
12+
select GENERIC_MSI_IRQ
1213
select REMOTEPROC
1314
select RPMSG
1415
help

drivers/cdx/controller/cdx_controller.c

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <linux/platform_device.h>
1010
#include <linux/slab.h>
1111
#include <linux/cdx/cdx_bus.h>
12+
#include <linux/irqdomain.h>
1213

1314
#include "cdx_controller.h"
1415
#include "../cdx.h"
@@ -60,16 +61,29 @@ static int cdx_configure_device(struct cdx_controller *cdx,
6061
u8 bus_num, u8 dev_num,
6162
struct cdx_device_config *dev_config)
6263
{
64+
u16 msi_index;
6365
int ret = 0;
66+
u32 data;
67+
u64 addr;
6468

6569
switch (dev_config->type) {
70+
case CDX_DEV_MSI_CONF:
71+
msi_index = dev_config->msi.msi_index;
72+
data = dev_config->msi.data;
73+
addr = dev_config->msi.addr;
74+
75+
ret = cdx_mcdi_write_msi(cdx->priv, bus_num, dev_num, msi_index, addr, data);
76+
break;
6677
case CDX_DEV_RESET_CONF:
6778
ret = cdx_mcdi_reset_device(cdx->priv, bus_num, dev_num);
6879
break;
6980
case CDX_DEV_BUS_MASTER_CONF:
7081
ret = cdx_mcdi_bus_master_enable(cdx->priv, bus_num, dev_num,
7182
dev_config->bus_master_enable);
7283
break;
84+
case CDX_DEV_MSI_ENABLE:
85+
ret = cdx_mcdi_msi_enable(cdx->priv, bus_num, dev_num, dev_config->msi_enable);
86+
break;
7387
default:
7488
ret = -EINVAL;
7589
}
@@ -178,6 +192,14 @@ static int xlnx_cdx_probe(struct platform_device *pdev)
178192
cdx->priv = cdx_mcdi;
179193
cdx->ops = &cdx_ops;
180194

195+
/* Create MSI domain */
196+
cdx->msi_domain = cdx_msi_domain_init(&pdev->dev);
197+
if (!cdx->msi_domain) {
198+
dev_err(&pdev->dev, "cdx_msi_domain_init() failed");
199+
ret = -ENODEV;
200+
goto cdx_msi_fail;
201+
}
202+
181203
ret = cdx_setup_rpmsg(pdev);
182204
if (ret) {
183205
if (ret != -EPROBE_DEFER)
@@ -189,6 +211,8 @@ static int xlnx_cdx_probe(struct platform_device *pdev)
189211
return 0;
190212

191213
cdx_rpmsg_fail:
214+
irq_domain_remove(cdx->msi_domain);
215+
cdx_msi_fail:
192216
kfree(cdx);
193217
cdx_alloc_fail:
194218
cdx_mcdi_finish(cdx_mcdi);
@@ -205,6 +229,7 @@ static int xlnx_cdx_remove(struct platform_device *pdev)
205229

206230
cdx_destroy_rpmsg(pdev);
207231

232+
irq_domain_remove(cdx->msi_domain);
208233
kfree(cdx);
209234

210235
cdx_mcdi_finish(cdx_mcdi);

0 commit comments

Comments
 (0)