Skip to content

Commit a9956d3

Browse files
ilantayariSaeed Mahameed
ilantayari
authored and
Saeed Mahameed
committed
net/mlx5: FPGA, Add SBU infrastructure
Add interface to initialize and interact with Innova FPGA SBU connections. A client driver may use these functions to set up a high-speed DMA connection with its SBU hardware logic, and send/receive messages over this connection. A later patch in this patchset will make use of these functions for Innova IPSec offload in mlx5 Ethernet driver. Add commands to retrieve Innova FPGA SBU capabilities, and to read/write Innova FPGA configuration space registers and memory, over internal I2C. At high level, the FPGA configuration space is divided such: 0x00000000 - 0x007fffff is reserved for the SBU 0x00800000 - 0xffffffff is reserved for the Shell 0x400000000 - ... is DDR memory A later patchset will add support for accessing FPGA CrSpace and memory over a high-speed connection. This is the reason for the ACCESS_TYPE enumeration, which currently only supports I2C. Signed-off-by: Ilan Tayari <[email protected]> Signed-off-by: Saeed Mahameed <[email protected]>
1 parent c43051d commit a9956d3

File tree

9 files changed

+349
-1
lines changed

9 files changed

+349
-1
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
66
mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
77
fs_counters.o rl.o lag.o dev.o lib/gid.o
88

9-
mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o
9+
mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o
1010

1111
mlx5_core-$(CONFIG_MLX5_CORE_EN) += wq.o eswitch.o eswitch_offloads.o \
1212
en_main.o en_common.o en_fs.o en_ethtool.o en_tx.o \

‎drivers/net/ethernet/mellanox/mlx5/core/fpga/cmd.c‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,39 @@
3838
#include "mlx5_core.h"
3939
#include "fpga/cmd.h"
4040

41+
#define MLX5_FPGA_ACCESS_REG_SZ (MLX5_ST_SZ_DW(fpga_access_reg) + \
42+
MLX5_FPGA_ACCESS_REG_SIZE_MAX)
43+
44+
int mlx5_fpga_access_reg(struct mlx5_core_dev *dev, u8 size, u64 addr,
45+
void *buf, bool write)
46+
{
47+
u32 in[MLX5_FPGA_ACCESS_REG_SZ] = {0};
48+
u32 out[MLX5_FPGA_ACCESS_REG_SZ];
49+
int err;
50+
51+
if (size & 3)
52+
return -EINVAL;
53+
if (addr & 3)
54+
return -EINVAL;
55+
if (size > MLX5_FPGA_ACCESS_REG_SIZE_MAX)
56+
return -EINVAL;
57+
58+
MLX5_SET(fpga_access_reg, in, size, size);
59+
MLX5_SET64(fpga_access_reg, in, address, addr);
60+
if (write)
61+
memcpy(MLX5_ADDR_OF(fpga_access_reg, in, data), buf, size);
62+
63+
err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
64+
MLX5_REG_FPGA_ACCESS_REG, 0, write);
65+
if (err)
66+
return err;
67+
68+
if (!write)
69+
memcpy(buf, MLX5_ADDR_OF(fpga_access_reg, out, data), size);
70+
71+
return 0;
72+
}
73+
4174
int mlx5_fpga_caps(struct mlx5_core_dev *dev, u32 *caps)
4275
{
4376
u32 in[MLX5_ST_SZ_DW(fpga_cap)] = {0};
@@ -58,6 +91,38 @@ int mlx5_fpga_ctrl_op(struct mlx5_core_dev *dev, u8 op)
5891
MLX5_REG_FPGA_CTRL, 0, true);
5992
}
6093

94+
int mlx5_fpga_sbu_caps(struct mlx5_core_dev *dev, void *caps, int size)
95+
{
96+
unsigned int cap_size = MLX5_CAP_FPGA(dev, sandbox_extended_caps_len);
97+
u64 addr = MLX5_CAP64_FPGA(dev, sandbox_extended_caps_addr);
98+
unsigned int read;
99+
int ret = 0;
100+
101+
if (cap_size > size) {
102+
mlx5_core_warn(dev, "Not enough buffer %u for FPGA SBU caps %u",
103+
size, cap_size);
104+
return -EINVAL;
105+
}
106+
107+
while (cap_size > 0) {
108+
read = min_t(unsigned int, cap_size,
109+
MLX5_FPGA_ACCESS_REG_SIZE_MAX);
110+
111+
ret = mlx5_fpga_access_reg(dev, read, addr, caps, false);
112+
if (ret) {
113+
mlx5_core_warn(dev, "Error reading FPGA SBU caps %u bytes at address 0x%llx: %d",
114+
read, addr, ret);
115+
return ret;
116+
}
117+
118+
cap_size -= read;
119+
addr += read;
120+
caps += read;
121+
}
122+
123+
return ret;
124+
}
125+
61126
int mlx5_fpga_query(struct mlx5_core_dev *dev, struct mlx5_fpga_query *query)
62127
{
63128
u32 in[MLX5_ST_SZ_DW(fpga_ctrl)] = {0};

‎drivers/net/ethernet/mellanox/mlx5/core/fpga/cmd.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ struct mlx5_fpga_qp_counters {
6868
int mlx5_fpga_caps(struct mlx5_core_dev *dev, u32 *caps);
6969
int mlx5_fpga_query(struct mlx5_core_dev *dev, struct mlx5_fpga_query *query);
7070
int mlx5_fpga_ctrl_op(struct mlx5_core_dev *dev, u8 op);
71+
int mlx5_fpga_access_reg(struct mlx5_core_dev *dev, u8 size, u64 addr,
72+
void *buf, bool write);
73+
int mlx5_fpga_sbu_caps(struct mlx5_core_dev *dev, void *caps, int size);
7174

7275
int mlx5_fpga_create_qp(struct mlx5_core_dev *dev, void *fpga_qpc,
7376
u32 *fpga_qpn);

‎drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.c‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
* Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3+
*
4+
* This software is available to you under a choice of one of two
5+
* licenses. You may choose to be licensed under the terms of the GNU
6+
* General Public License (GPL) Version 2, available from the file
7+
* COPYING in the main directory of this source tree, or the
8+
* OpenIB.org BSD license below:
9+
*
10+
* Redistribution and use in source and binary forms, with or
11+
* without modification, are permitted provided that the following
12+
* conditions are met:
13+
*
14+
* - Redistributions of source code must retain the above
15+
* copyright notice, this list of conditions and the following
16+
* disclaimer.
17+
*
18+
* - Redistributions in binary form must reproduce the above
19+
* copyright notice, this list of conditions and the following
20+
* disclaimer in the documentation and/or other materials
21+
* provided with the distribution.
22+
*
23+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28+
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30+
* SOFTWARE.
31+
*
32+
*/
33+
34+
#include <linux/mlx5/device.h>
35+
36+
#include "fpga/core.h"
37+
#include "fpga/conn.h"
38+
#include "fpga/sdk.h"
39+
40+
struct mlx5_fpga_conn *
41+
mlx5_fpga_sbu_conn_create(struct mlx5_fpga_device *fdev,
42+
struct mlx5_fpga_conn_attr *attr)
43+
{
44+
return mlx5_fpga_conn_create(fdev, attr, MLX5_FPGA_QPC_QP_TYPE_SANDBOX_QP);
45+
}
46+
EXPORT_SYMBOL(mlx5_fpga_sbu_conn_create);
47+
48+
void mlx5_fpga_sbu_conn_destroy(struct mlx5_fpga_conn *conn)
49+
{
50+
mlx5_fpga_conn_destroy(conn);
51+
}
52+
EXPORT_SYMBOL(mlx5_fpga_sbu_conn_destroy);
53+
54+
int mlx5_fpga_sbu_conn_sendmsg(struct mlx5_fpga_conn *conn,
55+
struct mlx5_fpga_dma_buf *buf)
56+
{
57+
return mlx5_fpga_conn_send(conn, buf);
58+
}
59+
EXPORT_SYMBOL(mlx5_fpga_sbu_conn_sendmsg);
60+
61+
static int mlx5_fpga_mem_read_i2c(struct mlx5_fpga_device *fdev, size_t size,
62+
u64 addr, u8 *buf)
63+
{
64+
size_t max_size = MLX5_FPGA_ACCESS_REG_SIZE_MAX;
65+
size_t bytes_done = 0;
66+
u8 actual_size;
67+
int err;
68+
69+
if (!fdev->mdev)
70+
return -ENOTCONN;
71+
72+
while (bytes_done < size) {
73+
actual_size = min(max_size, (size - bytes_done));
74+
75+
err = mlx5_fpga_access_reg(fdev->mdev, actual_size,
76+
addr + bytes_done,
77+
buf + bytes_done, false);
78+
if (err) {
79+
mlx5_fpga_err(fdev, "Failed to read over I2C: %d\n",
80+
err);
81+
break;
82+
}
83+
84+
bytes_done += actual_size;
85+
}
86+
87+
return err;
88+
}
89+
90+
static int mlx5_fpga_mem_write_i2c(struct mlx5_fpga_device *fdev, size_t size,
91+
u64 addr, u8 *buf)
92+
{
93+
size_t max_size = MLX5_FPGA_ACCESS_REG_SIZE_MAX;
94+
size_t bytes_done = 0;
95+
u8 actual_size;
96+
int err;
97+
98+
if (!fdev->mdev)
99+
return -ENOTCONN;
100+
101+
while (bytes_done < size) {
102+
actual_size = min(max_size, (size - bytes_done));
103+
104+
err = mlx5_fpga_access_reg(fdev->mdev, actual_size,
105+
addr + bytes_done,
106+
buf + bytes_done, true);
107+
if (err) {
108+
mlx5_fpga_err(fdev, "Failed to write FPGA crspace\n");
109+
break;
110+
}
111+
112+
bytes_done += actual_size;
113+
}
114+
115+
return err;
116+
}
117+
118+
int mlx5_fpga_mem_read(struct mlx5_fpga_device *fdev, size_t size, u64 addr,
119+
void *buf, enum mlx5_fpga_access_type access_type)
120+
{
121+
int ret;
122+
123+
switch (access_type) {
124+
case MLX5_FPGA_ACCESS_TYPE_I2C:
125+
ret = mlx5_fpga_mem_read_i2c(fdev, size, addr, buf);
126+
if (ret)
127+
return ret;
128+
break;
129+
default:
130+
mlx5_fpga_warn(fdev, "Unexpected read access_type %u\n",
131+
access_type);
132+
return -EACCES;
133+
}
134+
135+
return size;
136+
}
137+
EXPORT_SYMBOL(mlx5_fpga_mem_read);
138+
139+
int mlx5_fpga_mem_write(struct mlx5_fpga_device *fdev, size_t size, u64 addr,
140+
void *buf, enum mlx5_fpga_access_type access_type)
141+
{
142+
int ret;
143+
144+
switch (access_type) {
145+
case MLX5_FPGA_ACCESS_TYPE_I2C:
146+
ret = mlx5_fpga_mem_write_i2c(fdev, size, addr, buf);
147+
if (ret)
148+
return ret;
149+
break;
150+
default:
151+
mlx5_fpga_warn(fdev, "Unexpected write access_type %u\n",
152+
access_type);
153+
return -EACCES;
154+
}
155+
156+
return size;
157+
}
158+
EXPORT_SYMBOL(mlx5_fpga_mem_write);
159+
160+
int mlx5_fpga_get_sbu_caps(struct mlx5_fpga_device *fdev, int size, void *buf)
161+
{
162+
return mlx5_fpga_sbu_caps(fdev->mdev, buf, size);
163+
}
164+
EXPORT_SYMBOL(mlx5_fpga_get_sbu_caps);

‎drivers/net/ethernet/mellanox/mlx5/core/fpga/sdk.h‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@
4242
* This header defines the in-kernel API for Innova FPGA client drivers.
4343
*/
4444

45+
enum mlx5_fpga_access_type {
46+
MLX5_FPGA_ACCESS_TYPE_I2C = 0x0,
47+
MLX5_FPGA_ACCESS_TYPE_DONTCARE = 0x0,
48+
};
49+
4550
struct mlx5_fpga_conn;
4651
struct mlx5_fpga_device;
4752

@@ -103,4 +108,97 @@ struct mlx5_fpga_conn_attr {
103108
void *cb_arg;
104109
};
105110

111+
/**
112+
* mlx5_fpga_sbu_conn_create() - Initialize a new FPGA SBU connection
113+
* @fdev: The FPGA device
114+
* @attr: Attributes of the new connection
115+
*
116+
* Sets up a new FPGA SBU connection with the specified attributes.
117+
* The receive callback function may be called for incoming messages even
118+
* before this function returns.
119+
*
120+
* The caller must eventually destroy the connection by calling
121+
* mlx5_fpga_sbu_conn_destroy.
122+
*
123+
* Return: A new connection, or ERR_PTR() error value otherwise.
124+
*/
125+
struct mlx5_fpga_conn *
126+
mlx5_fpga_sbu_conn_create(struct mlx5_fpga_device *fdev,
127+
struct mlx5_fpga_conn_attr *attr);
128+
129+
/**
130+
* mlx5_fpga_sbu_conn_destroy() - Destroy an FPGA SBU connection
131+
* @conn: The FPGA SBU connection to destroy
132+
*
133+
* Cleans up an FPGA SBU connection which was previously created with
134+
* mlx5_fpga_sbu_conn_create.
135+
*/
136+
void mlx5_fpga_sbu_conn_destroy(struct mlx5_fpga_conn *conn);
137+
138+
/**
139+
* mlx5_fpga_sbu_conn_sendmsg() - Queue the transmission of a packet
140+
* @fdev: An FPGA SBU connection
141+
* @buf: The packet buffer
142+
*
143+
* Queues a packet for transmission over an FPGA SBU connection.
144+
* The buffer should not be modified or freed until completion.
145+
* Upon completion, the buf's complete() callback is invoked, indicating the
146+
* success or error status of the transmission.
147+
*
148+
* Return: 0 if successful, or an error value otherwise.
149+
*/
150+
int mlx5_fpga_sbu_conn_sendmsg(struct mlx5_fpga_conn *conn,
151+
struct mlx5_fpga_dma_buf *buf);
152+
153+
/**
154+
* mlx5_fpga_mem_read() - Read from FPGA memory address space
155+
* @fdev: The FPGA device
156+
* @size: Size of chunk to read, in bytes
157+
* @addr: Starting address to read from, in FPGA address space
158+
* @buf: Buffer to read into
159+
* @access_type: Method for reading
160+
*
161+
* Reads from the specified address into the specified buffer.
162+
* The address may point to configuration space or to DDR.
163+
* Large reads may be performed internally as several non-atomic operations.
164+
* This function may sleep, so should not be called from atomic contexts.
165+
*
166+
* Return: 0 if successful, or an error value otherwise.
167+
*/
168+
int mlx5_fpga_mem_read(struct mlx5_fpga_device *fdev, size_t size, u64 addr,
169+
void *buf, enum mlx5_fpga_access_type access_type);
170+
171+
/**
172+
* mlx5_fpga_mem_write() - Write to FPGA memory address space
173+
* @fdev: The FPGA device
174+
* @size: Size of chunk to write, in bytes
175+
* @addr: Starting address to write to, in FPGA address space
176+
* @buf: Buffer which contains data to write
177+
* @access_type: Method for writing
178+
*
179+
* Writes the specified buffer data to FPGA memory at the specified address.
180+
* The address may point to configuration space or to DDR.
181+
* Large writes may be performed internally as several non-atomic operations.
182+
* This function may sleep, so should not be called from atomic contexts.
183+
*
184+
* Return: 0 if successful, or an error value otherwise.
185+
*/
186+
int mlx5_fpga_mem_write(struct mlx5_fpga_device *fdev, size_t size, u64 addr,
187+
void *buf, enum mlx5_fpga_access_type access_type);
188+
189+
/**
190+
* mlx5_fpga_get_sbu_caps() - Read the SBU capabilities
191+
* @fdev: The FPGA device
192+
* @size: Size of the buffer to read into
193+
* @buf: Buffer to read the capabilities into
194+
*
195+
* Reads the FPGA SBU capabilities into the specified buffer.
196+
* The format of the capabilities buffer is SBU-dependent.
197+
*
198+
* Return: 0 if successful
199+
* -EINVAL if the buffer is not large enough to contain SBU caps
200+
* or any other error value otherwise.
201+
*/
202+
int mlx5_fpga_get_sbu_caps(struct mlx5_fpga_device *fdev, int size, void *buf);
203+
106204
#endif /* MLX5_FPGA_SDK_H */

‎include/linux/mlx5/device.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,6 +1103,9 @@ enum mlx5_mcam_feature_groups {
11031103
#define MLX5_CAP_FPGA(mdev, cap) \
11041104
MLX5_GET(fpga_cap, (mdev)->caps.hca_cur[MLX5_CAP_FPGA], cap)
11051105

1106+
#define MLX5_CAP64_FPGA(mdev, cap) \
1107+
MLX5_GET64(fpga_cap, (mdev)->caps.hca_cur[MLX5_CAP_FPGA], cap)
1108+
11061109
enum {
11071110
MLX5_CMD_STAT_OK = 0x0,
11081111
MLX5_CMD_STAT_INT_ERR = 0x1,

‎include/linux/mlx5/driver.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ enum {
111111
MLX5_REG_DCBX_APP = 0x4021,
112112
MLX5_REG_FPGA_CAP = 0x4022,
113113
MLX5_REG_FPGA_CTRL = 0x4023,
114+
MLX5_REG_FPGA_ACCESS_REG = 0x4024,
114115
MLX5_REG_PCAP = 0x5001,
115116
MLX5_REG_PMTU = 0x5003,
116117
MLX5_REG_PTYS = 0x5004,

‎include/linux/mlx5/mlx5_ifc.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8309,6 +8309,7 @@ union mlx5_ifc_ports_control_registers_document_bits {
83098309
struct mlx5_ifc_sltp_reg_bits sltp_reg;
83108310
struct mlx5_ifc_mtpps_reg_bits mtpps_reg;
83118311
struct mlx5_ifc_mtppse_reg_bits mtppse_reg;
8312+
struct mlx5_ifc_fpga_access_reg_bits fpga_access_reg;
83128313
struct mlx5_ifc_fpga_ctrl_bits fpga_ctrl_bits;
83138314
struct mlx5_ifc_fpga_cap_bits fpga_cap_bits;
83148315
struct mlx5_ifc_mcqi_reg_bits mcqi_reg;

0 commit comments

Comments
 (0)