Skip to content

Commit 9c6a595

Browse files
davem330davem330
committed
Merge branch 'octeontx2-pf-mbox-fixes'
Subbaraya Sundeep says: ==================== octeontx2-pf: RVU Mailbox fixes This patchset fixes the problems related to RVU mailbox. During long run tests some times VF commands like setting MTU or toggling interface fails because VF mailbox is timedout waiting for response from PF. Below are the fixes Patch 1: There are two types of messages in RVU mailbox namely up and down messages. Down messages are synchronous messages where a PF/VF sends a message to AF and AF replies back with response. UP messages are notifications and are asynchronous like AF sending link events to PF. When VF sends a down message to PF, PF forwards to AF and sends the response from AF back to VF. PF has to forward VF messages since there is no path in hardware for VF to send directly to AF. There is one mailbox interrupt from AF to PF when raised could mean two scenarios one is where AF sending reply to PF for a down message sent by PF and another one is AF sending up message asynchronously when link changed for that PF. Receiving the up message interrupt while PF is in middle of forwarding down message causes mailbox errors. Fix this by receiver detecting the type of message from the mbox data register set by sender. Patch 2: During VF driver remove, VF has to wait until last message is completed and then turn off mailbox interrupts from PF. Patch 3: Do not use ordered workqueue for message processing since multiple works are queued simultaneously by all the VFs and PF link UP messages. Patch 4: When sending link event to VF by PF check whether VF is really up to receive this message. Patch 5: In AF driver, use separate interrupt handlers for the AF-VF interrupt and AF-PF interrupt. Sometimes both interrupts are raised to two CPUs at same time and both CPUs execute same function at same time corrupting the data. v2 changes: Added missing mutex unlock in error path in patch 1 Refactored if else logic in patch 1 as suggested by Paolo Abeni ==================== Signed-off-by: David S. Miller <[email protected]>
2 parents 94e3ca2 + 50e60de commit 9c6a595

File tree

10 files changed

+225
-88
lines changed

10 files changed

+225
-88
lines changed

‎drivers/net/ethernet/marvell/octeontx2/af/mbox.c

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,11 +214,12 @@ int otx2_mbox_busy_poll_for_rsp(struct otx2_mbox *mbox, int devid)
214214
}
215215
EXPORT_SYMBOL(otx2_mbox_busy_poll_for_rsp);
216216

217-
void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid)
217+
static void otx2_mbox_msg_send_data(struct otx2_mbox *mbox, int devid, u64 data)
218218
{
219219
struct otx2_mbox_dev *mdev = &mbox->dev[devid];
220220
struct mbox_hdr *tx_hdr, *rx_hdr;
221221
void *hw_mbase = mdev->hwbase;
222+
u64 intr_val;
222223

223224
tx_hdr = hw_mbase + mbox->tx_start;
224225
rx_hdr = hw_mbase + mbox->rx_start;
@@ -254,14 +255,52 @@ void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid)
254255

255256
spin_unlock(&mdev->mbox_lock);
256257

258+
/* Check if interrupt pending */
259+
intr_val = readq((void __iomem *)mbox->reg_base +
260+
(mbox->trigger | (devid << mbox->tr_shift)));
261+
262+
intr_val |= data;
257263
/* The interrupt should be fired after num_msgs is written
258264
* to the shared memory
259265
*/
260-
writeq(1, (void __iomem *)mbox->reg_base +
266+
writeq(intr_val, (void __iomem *)mbox->reg_base +
261267
(mbox->trigger | (devid << mbox->tr_shift)));
262268
}
269+
270+
void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid)
271+
{
272+
otx2_mbox_msg_send_data(mbox, devid, MBOX_DOWN_MSG);
273+
}
263274
EXPORT_SYMBOL(otx2_mbox_msg_send);
264275

276+
void otx2_mbox_msg_send_up(struct otx2_mbox *mbox, int devid)
277+
{
278+
otx2_mbox_msg_send_data(mbox, devid, MBOX_UP_MSG);
279+
}
280+
EXPORT_SYMBOL(otx2_mbox_msg_send_up);
281+
282+
bool otx2_mbox_wait_for_zero(struct otx2_mbox *mbox, int devid)
283+
{
284+
u64 data;
285+
286+
data = readq((void __iomem *)mbox->reg_base +
287+
(mbox->trigger | (devid << mbox->tr_shift)));
288+
289+
/* If data is non-zero wait for ~1ms and return to caller
290+
* whether data has changed to zero or not after the wait.
291+
*/
292+
if (!data)
293+
return true;
294+
295+
usleep_range(950, 1000);
296+
297+
data = readq((void __iomem *)mbox->reg_base +
298+
(mbox->trigger | (devid << mbox->tr_shift)));
299+
300+
return data == 0;
301+
}
302+
EXPORT_SYMBOL(otx2_mbox_wait_for_zero);
303+
265304
struct mbox_msghdr *otx2_mbox_alloc_msg_rsp(struct otx2_mbox *mbox, int devid,
266305
int size, int size_rsp)
267306
{

‎drivers/net/ethernet/marvell/octeontx2/af/mbox.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616

1717
#define MBOX_SIZE SZ_64K
1818

19+
#define MBOX_DOWN_MSG 1
20+
#define MBOX_UP_MSG 2
21+
1922
/* AF/PF: PF initiated, PF/VF VF initiated */
2023
#define MBOX_DOWN_RX_START 0
2124
#define MBOX_DOWN_RX_SIZE (46 * SZ_1K)
@@ -101,6 +104,7 @@ int otx2_mbox_regions_init(struct otx2_mbox *mbox, void __force **hwbase,
101104
struct pci_dev *pdev, void __force *reg_base,
102105
int direction, int ndevs, unsigned long *bmap);
103106
void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid);
107+
void otx2_mbox_msg_send_up(struct otx2_mbox *mbox, int devid);
104108
int otx2_mbox_wait_for_rsp(struct otx2_mbox *mbox, int devid);
105109
int otx2_mbox_busy_poll_for_rsp(struct otx2_mbox *mbox, int devid);
106110
struct mbox_msghdr *otx2_mbox_alloc_msg_rsp(struct otx2_mbox *mbox, int devid,
@@ -118,6 +122,8 @@ static inline struct mbox_msghdr *otx2_mbox_alloc_msg(struct otx2_mbox *mbox,
118122
return otx2_mbox_alloc_msg_rsp(mbox, devid, size, 0);
119123
}
120124

125+
bool otx2_mbox_wait_for_zero(struct otx2_mbox *mbox, int devid);
126+
121127
/* Mailbox message types */
122128
#define MBOX_MSG_MASK 0xFFFF
123129
#define MBOX_MSG_INVALID 0xFFFE

‎drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -121,24 +121,29 @@ int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event)
121121
static int mcs_notify_pfvf(struct mcs_intr_event *event, struct rvu *rvu)
122122
{
123123
struct mcs_intr_info *req;
124-
int err, pf;
124+
int pf;
125125

126126
pf = rvu_get_pf(event->pcifunc);
127127

128+
mutex_lock(&rvu->mbox_lock);
129+
128130
req = otx2_mbox_alloc_msg_mcs_intr_notify(rvu, pf);
129-
if (!req)
131+
if (!req) {
132+
mutex_unlock(&rvu->mbox_lock);
130133
return -ENOMEM;
134+
}
131135

132136
req->mcs_id = event->mcs_id;
133137
req->intr_mask = event->intr_mask;
134138
req->sa_id = event->sa_id;
135139
req->hdr.pcifunc = event->pcifunc;
136140
req->lmac_id = event->lmac_id;
137141

138-
otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pf);
139-
err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pf);
140-
if (err)
141-
dev_warn(rvu->dev, "MCS notification to pf %d failed\n", pf);
142+
otx2_mbox_wait_for_zero(&rvu->afpf_wq_info.mbox_up, pf);
143+
144+
otx2_mbox_msg_send_up(&rvu->afpf_wq_info.mbox_up, pf);
145+
146+
mutex_unlock(&rvu->mbox_lock);
142147

143148
return 0;
144149
}

‎drivers/net/ethernet/marvell/octeontx2/af/rvu.c

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2119,7 +2119,7 @@ MBOX_MESSAGES
21192119
}
21202120
}
21212121

2122-
static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
2122+
static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll)
21232123
{
21242124
struct rvu *rvu = mwork->rvu;
21252125
int offset, err, id, devid;
@@ -2186,22 +2186,28 @@ static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
21862186
}
21872187
mw->mbox_wrk[devid].num_msgs = 0;
21882188

2189+
if (poll)
2190+
otx2_mbox_wait_for_zero(mbox, devid);
2191+
21892192
/* Send mbox responses to VF/PF */
21902193
otx2_mbox_msg_send(mbox, devid);
21912194
}
21922195

21932196
static inline void rvu_afpf_mbox_handler(struct work_struct *work)
21942197
{
21952198
struct rvu_work *mwork = container_of(work, struct rvu_work, work);
2199+
struct rvu *rvu = mwork->rvu;
21962200

2197-
__rvu_mbox_handler(mwork, TYPE_AFPF);
2201+
mutex_lock(&rvu->mbox_lock);
2202+
__rvu_mbox_handler(mwork, TYPE_AFPF, true);
2203+
mutex_unlock(&rvu->mbox_lock);
21982204
}
21992205

22002206
static inline void rvu_afvf_mbox_handler(struct work_struct *work)
22012207
{
22022208
struct rvu_work *mwork = container_of(work, struct rvu_work, work);
22032209

2204-
__rvu_mbox_handler(mwork, TYPE_AFVF);
2210+
__rvu_mbox_handler(mwork, TYPE_AFVF, false);
22052211
}
22062212

22072213
static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
@@ -2376,6 +2382,8 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
23762382
}
23772383
}
23782384

2385+
mutex_init(&rvu->mbox_lock);
2386+
23792387
mbox_regions = kcalloc(num, sizeof(void *), GFP_KERNEL);
23802388
if (!mbox_regions) {
23812389
err = -ENOMEM;
@@ -2525,10 +2533,9 @@ static void rvu_queue_work(struct mbox_wq_info *mw, int first,
25252533
}
25262534
}
25272535

2528-
static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
2536+
static irqreturn_t rvu_mbox_pf_intr_handler(int irq, void *rvu_irq)
25292537
{
25302538
struct rvu *rvu = (struct rvu *)rvu_irq;
2531-
int vfs = rvu->vfs;
25322539
u64 intr;
25332540

25342541
intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
@@ -2542,6 +2549,18 @@ static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
25422549

25432550
rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
25442551

2552+
return IRQ_HANDLED;
2553+
}
2554+
2555+
static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
2556+
{
2557+
struct rvu *rvu = (struct rvu *)rvu_irq;
2558+
int vfs = rvu->vfs;
2559+
u64 intr;
2560+
2561+
/* Sync with mbox memory region */
2562+
rmb();
2563+
25452564
/* Handle VF interrupts */
25462565
if (vfs > 64) {
25472566
intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
@@ -2886,7 +2905,7 @@ static int rvu_register_interrupts(struct rvu *rvu)
28862905
/* Register mailbox interrupt handler */
28872906
sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");
28882907
ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),
2889-
rvu_mbox_intr_handler, 0,
2908+
rvu_mbox_pf_intr_handler, 0,
28902909
&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);
28912910
if (ret) {
28922911
dev_err(rvu->dev,

‎drivers/net/ethernet/marvell/octeontx2/af/rvu.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,8 @@ struct rvu {
591591
spinlock_t mcs_intrq_lock;
592592
/* CPT interrupt lock */
593593
spinlock_t cpt_intr_lock;
594+
595+
struct mutex mbox_lock; /* Serialize mbox up and down msgs */
594596
};
595597

596598
static inline void rvu_write64(struct rvu *rvu, u64 block, u64 offset, u64 val)

‎drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ static void cgx_notify_pfs(struct cgx_link_event *event, struct rvu *rvu)
232232
struct cgx_link_user_info *linfo;
233233
struct cgx_link_info_msg *msg;
234234
unsigned long pfmap;
235-
int err, pfid;
235+
int pfid;
236236

237237
linfo = &event->link_uinfo;
238238
pfmap = cgxlmac_to_pfmap(rvu, event->cgx_id, event->lmac_id);
@@ -255,16 +255,22 @@ static void cgx_notify_pfs(struct cgx_link_event *event, struct rvu *rvu)
255255
continue;
256256
}
257257

258+
mutex_lock(&rvu->mbox_lock);
259+
258260
/* Send mbox message to PF */
259261
msg = otx2_mbox_alloc_msg_cgx_link_event(rvu, pfid);
260-
if (!msg)
262+
if (!msg) {
263+
mutex_unlock(&rvu->mbox_lock);
261264
continue;
265+
}
266+
262267
msg->link_info = *linfo;
263-
otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pfid);
264-
err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pfid);
265-
if (err)
266-
dev_warn(rvu->dev, "notification to pf %d failed\n",
267-
pfid);
268+
269+
otx2_mbox_wait_for_zero(&rvu->afpf_wq_info.mbox_up, pfid);
270+
271+
otx2_mbox_msg_send_up(&rvu->afpf_wq_info.mbox_up, pfid);
272+
273+
mutex_unlock(&rvu->mbox_lock);
268274
} while (pfmap);
269275
}
270276

‎drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1592,7 +1592,7 @@ int otx2_detach_resources(struct mbox *mbox)
15921592
detach->partial = false;
15931593

15941594
/* Send detach request to AF */
1595-
otx2_mbox_msg_send(&mbox->mbox, 0);
1595+
otx2_sync_mbox_msg(mbox);
15961596
mutex_unlock(&mbox->lock);
15971597
return 0;
15981598
}

‎drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -815,7 +815,7 @@ static inline int otx2_sync_mbox_up_msg(struct mbox *mbox, int devid)
815815

816816
if (!otx2_mbox_nonempty(&mbox->mbox_up, devid))
817817
return 0;
818-
otx2_mbox_msg_send(&mbox->mbox_up, devid);
818+
otx2_mbox_msg_send_up(&mbox->mbox_up, devid);
819819
err = otx2_mbox_wait_for_rsp(&mbox->mbox_up, devid);
820820
if (err)
821821
return err;

0 commit comments

Comments
 (0)