From 4f1b4c8713be92d65c7f49c6413fead6658c8b69 Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Mon, 17 Aug 2020 19:59:49 -0700 Subject: [PATCH 1/5] xdma: driver update --- sdk/linux_kernel_drivers/xdma/Makefile | 2 +- sdk/linux_kernel_drivers/xdma/cdev_bypass.c | 33 +- sdk/linux_kernel_drivers/xdma/cdev_ctrl.c | 53 +- sdk/linux_kernel_drivers/xdma/cdev_ctrl.h | 19 +- sdk/linux_kernel_drivers/xdma/cdev_events.c | 6 +- sdk/linux_kernel_drivers/xdma/cdev_sgdma.c | 561 +++- sdk/linux_kernel_drivers/xdma/cdev_sgdma.h | 24 +- sdk/linux_kernel_drivers/xdma/cdev_xvc.c | 75 +- sdk/linux_kernel_drivers/xdma/cdev_xvc.h | 9 +- sdk/linux_kernel_drivers/xdma/libxdma.c | 2824 +++++++++++++------ sdk/linux_kernel_drivers/xdma/libxdma.h | 99 +- sdk/linux_kernel_drivers/xdma/libxdma_api.h | 26 +- sdk/linux_kernel_drivers/xdma/version.h | 7 +- sdk/linux_kernel_drivers/xdma/xdma_cdev.c | 228 +- sdk/linux_kernel_drivers/xdma/xdma_cdev.h | 7 +- sdk/linux_kernel_drivers/xdma/xdma_ioctl.h | 78 - sdk/linux_kernel_drivers/xdma/xdma_mod.c | 125 +- sdk/linux_kernel_drivers/xdma/xdma_mod.h | 27 +- sdk/linux_kernel_drivers/xdma/xdma_thread.c | 345 +++ sdk/linux_kernel_drivers/xdma/xdma_thread.h | 152 + 20 files changed, 3283 insertions(+), 1417 deletions(-) delete mode 100755 sdk/linux_kernel_drivers/xdma/xdma_ioctl.h create mode 100644 sdk/linux_kernel_drivers/xdma/xdma_thread.c create mode 100644 sdk/linux_kernel_drivers/xdma/xdma_thread.h diff --git a/sdk/linux_kernel_drivers/xdma/Makefile b/sdk/linux_kernel_drivers/xdma/Makefile index 182051b70..b3ad79d2f 100755 --- a/sdk/linux_kernel_drivers/xdma/Makefile +++ b/sdk/linux_kernel_drivers/xdma/Makefile @@ -32,7 +32,7 @@ EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS) #EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__ ifneq ($(KERNELRELEASE),) - $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o + $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o obj-m := $(TARGET_MODULE).o else BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build diff --git a/sdk/linux_kernel_drivers/xdma/cdev_bypass.c b/sdk/linux_kernel_drivers/xdma/cdev_bypass.c index 9ab445ea7..d38d3fac3 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_bypass.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_bypass.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,11 +21,10 @@ * Karen Xie * ******************************************************************************/ - #include "libxdma_api.h" #include "xdma_cdev.h" -#define write_register(v,mem,off) iowrite32(v, mem) +#define write_register(v, mem, off) iowrite32(v, mem) static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf, size_t *buf_offset, size_t buf_size) @@ -34,8 +33,15 @@ static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf, int copy_err; int rc = 0; - BUG_ON(!buf); - BUG_ON(!buf_offset); + if (!buf) { + pr_err("Invalid user buffer\n"); + return -EINVAL; + } + + if (!buf_offset) { + pr_err("Invalid user buffer offset\n"); + return -EINVAL; + } /* Fill user buffer with descriptor data */ for (i = 0; i < transfer->desc_num; i++) { @@ -76,7 +82,7 @@ static ssize_t char_bypass_read(struct file *file, char __user *buf, xdev = xcdev->xdev; engine = xcdev->engine; - dbg_sg("In char_bypass_read()\n"); + dbg_sg("In %s()\n", __func__); if (count & 3) { dbg_sg("Buffer size must be a multiple of 4 bytes\n"); @@ -119,7 +125,7 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf, struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; u32 desc_data; - u32 *bypass_addr; + void __iomem *bypass_addr; size_t buf_offset = 0; int rc = 0; int copy_err; @@ -145,18 +151,21 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf, return -ENODEV; } - dbg_sg("In char_bypass_write()\n"); + dbg_sg("In %s()\n", __func__); spin_lock(&engine->lock); /* Write descriptor data to the bypass BAR */ - bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx]; - bypass_addr += engine->bypass_offset; + bypass_addr = xdev->bar[xdev->bypass_bar_idx]; + bypass_addr = (void __iomem *)( + (u32 __iomem *)bypass_addr + engine->bypass_offset + ); while (buf_offset < count) { copy_err = copy_from_user(&desc_data, &buf[buf_offset], sizeof(u32)); if (!copy_err) { - write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset); + write_register(desc_data, bypass_addr, + bypass_addr - engine->bypass_offset); buf_offset += sizeof(u32); rc = buf_offset; } else { @@ -188,5 +197,5 @@ static const struct file_operations bypass_fops = { void cdev_bypass_init(struct xdma_cdev *xcdev) { - cdev_init(&xcdev->cdev, &bypass_fops); + cdev_init(&xcdev->cdev, &bypass_fops); } diff --git a/sdk/linux_kernel_drivers/xdma/cdev_ctrl.c b/sdk/linux_kernel_drivers/xdma/cdev_ctrl.c index 404bbd7fa..9fa7a3522 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_ctrl.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_ctrl.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include @@ -28,6 +29,12 @@ #include "xdma_cdev.h" #include "cdev_ctrl.h" +#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE +#define xlx_access_ok(X, Y, Z) access_ok(Y, Z) +#else +#define xlx_access_ok(X, Y, Z) access_ok(X, Y, Z) +#endif + /* * character device file operations for control bus (through control bridge) */ @@ -36,13 +43,13 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count, { struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data; struct xdma_dev *xdev; - void *reg; + void __iomem *reg; u32 w; int rv; rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; + return rv; xdev = xcdev->xdev; /* only 32-bit aligned and 32-bit multiples */ @@ -52,8 +59,8 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count, reg = xdev->bar[xcdev->bar] + *pos; //w = read_register(reg); w = ioread32(reg); - dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg, - (long)count, (int)*pos, w); + dbg_sg("%s(@%p, count=%ld, pos=%d) value = 0x%08x\n", + __func__, reg, (long)count, (int)*pos, w); rv = copy_to_user(buf, &w, 4); if (rv) dbg_sg("Copy to userspace failed but continuing\n"); @@ -67,13 +74,13 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf, { struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; struct xdma_dev *xdev; - void *reg; + void __iomem *reg; u32 w; int rv; rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; + return rv; xdev = xcdev->xdev; /* only 32-bit aligned and 32-bit multiples */ @@ -83,12 +90,11 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf, /* first address is BAR base plus file position offset */ reg = xdev->bar[xcdev->bar] + *pos; rv = copy_from_user(&w, buf, 4); - if (rv) { + if (rv) pr_info("copy from user failed %d/4, but continuing.\n", rv); - } - dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg, - (long)count, (int)*pos); + dbg_sg("%s(0x%08x @%p, count=%ld, pos=%d)\n", + __func__, w, reg, (long)count, (int)*pos); //write_register(w, reg); iowrite32(w, reg); *pos += 4; @@ -133,9 +139,13 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; - xdev = xcdev->xdev; + return rv; + xdev = xcdev->xdev; + if (!xdev) { + pr_info("cmd %u, xdev NULL.\n", cmd); + return -EINVAL; + } pr_info("cmd 0x%x, xdev 0x%p, pdev 0x%p.\n", cmd, xdev, xdev->pdev); if (_IOC_TYPE(cmd) != XDMA_IOC_MAGIC) { @@ -145,10 +155,10 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) } if (_IOC_DIR(cmd) & _IOC_READ) - result = !access_ok(VERIFY_WRITE, (void __user *)arg, + result = !xlx_access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd)); else if (_IOC_DIR(cmd) & _IOC_WRITE) - result = !access_ok(VERIFY_READ, (void __user *)arg, + result = !xlx_access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd)); if (result) { @@ -158,7 +168,7 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) switch (cmd) { case XDMA_IOCINFO: - if (copy_from_user((void *)&ioctl_obj, (void *) arg, + if (copy_from_user((void *)&ioctl_obj, (void __user *) arg, sizeof(struct xdma_ioc_base))) { pr_err("copy_from_user failed.\n"); return -EFAULT; @@ -169,20 +179,11 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ioctl_obj.magic, XDMA_XCL_MAGIC); return -ENOTTY; } - return version_ioctl(xcdev, (void __user *)arg); case XDMA_IOCOFFLINE: - if (!xdev) { - pr_info("cmd %u, xdev NULL.\n", cmd); - return -EINVAL; - } xdma_device_offline(xdev->pdev, xdev); break; case XDMA_IOCONLINE: - if (!xdev) { - pr_info("cmd %u, xdev NULL.\n", cmd); - return -EINVAL; - } xdma_device_online(xdev->pdev, xdev); break; default: @@ -205,7 +206,7 @@ int bridge_mmap(struct file *file, struct vm_area_struct *vma) rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; + return rv; xdev = xcdev->xdev; off = vma->vm_pgoff << PAGE_SHIFT; diff --git a/sdk/linux_kernel_drivers/xdma/cdev_ctrl.h b/sdk/linux_kernel_drivers/xdma/cdev_ctrl.h index 47e697cd6..e0a9047b6 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_ctrl.h +++ b/sdk/linux_kernel_drivers/xdma/cdev_ctrl.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef _XDMA_IOCALLS_POSIX_H_ #define _XDMA_IOCALLS_POSIX_H_ @@ -64,14 +65,14 @@ struct xdma_ioc_base { }; struct xdma_ioc_info { - struct xdma_ioc_base base; - unsigned short vendor; - unsigned short device; - unsigned short subsystem_vendor; - unsigned short subsystem_device; - unsigned int dma_engine_version; - unsigned int driver_version; - unsigned long long feature_id; + struct xdma_ioc_base base; + unsigned short vendor; + unsigned short device; + unsigned short subsystem_vendor; + unsigned short subsystem_device; + unsigned int dma_engine_version; + unsigned int driver_version; + unsigned long long feature_id; unsigned short domain; unsigned char bus; unsigned char dev; diff --git a/sdk/linux_kernel_drivers/xdma/cdev_events.c b/sdk/linux_kernel_drivers/xdma/cdev_events.c index 514aaf43b..2b468ed78 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_events.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_events.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -39,7 +39,7 @@ static ssize_t char_events_read(struct file *file, char __user *buf, rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; + return rv; user_irq = xcdev->user_irq; if (!user_irq) { pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); @@ -88,7 +88,7 @@ static unsigned int char_events_poll(struct file *file, poll_table *wait) rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) - return rv; + return rv; user_irq = xcdev->user_irq; if (!user_irq) { pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); diff --git a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c index 31854f92a..9b344356f 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,18 +21,113 @@ * Karen Xie * ******************************************************************************/ + #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ +#include #include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) +#include +#endif #include "libxdma_api.h" #include "xdma_cdev.h" #include "cdev_sgdma.h" +#include "xdma_thread.h" /* Module Parameters */ unsigned int sgdma_timeout = 10; module_param(sgdma_timeout, uint, 0644); MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec."); + +extern struct kmem_cache *cdev_cache; +static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write); + + +static void async_io_handler(unsigned long cb_hndl, int err) +{ + struct xdma_cdev *xcdev; + struct xdma_engine *engine; + struct xdma_dev *xdev; + struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; + struct cdev_async_io *caio = (struct cdev_async_io *)cb->private; + ssize_t numbytes = 0; + ssize_t res, res2; + int lock_stat; + int rv; + + if (caio == NULL) { + pr_err("Invalid work struct\n"); + return; + } + + xcdev = (struct xdma_cdev *)caio->iocb->ki_filp->private_data; + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return; + + /* Safeguarding for cancel requests */ + lock_stat = spin_trylock(&caio->lock); + if (!lock_stat) { + pr_err("caio lock not acquired\n"); + goto skip_dev_lock; + } + + if (false != caio->cancel) { + pr_err("skipping aio\n"); + goto skip_tran; + } + + engine = xcdev->engine; + xdev = xcdev->xdev; + + if (!err) + numbytes = xdma_xfer_completion((void *)cb, xdev, + engine->channel, cb->write, cb->ep_addr, + &cb->sgt, 0, sgdma_timeout * 1000); + + char_sgdma_unmap_user_buf(cb, cb->write); + + caio->res2 |= (err < 0) ? err : 0; + if (caio->res2) + caio->err_cnt++; + + caio->cmpl_cnt++; + caio->res += numbytes; + + if (caio->cmpl_cnt == caio->req_cnt) { + res = caio->res; + res2 = caio->res2; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) + caio->iocb->ki_complete(caio->iocb, res, res2); +#else + aio_complete(caio->iocb, res, res2); +#endif +skip_tran: + spin_unlock(&caio->lock); + kmem_cache_free(cdev_cache, caio); + kfree(cb); + return; + } + spin_unlock(&caio->lock); + return; + +skip_dev_lock: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) + caio->iocb->ki_complete(caio->iocb, numbytes, -EBUSY); +#else + aio_complete(caio->iocb, numbytes, -EBUSY); +#endif + kmem_cache_free(cdev_cache, caio); +} + + /* * character device file operations for SG DMA engine */ @@ -56,10 +151,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence) if (newpos < 0) return -EINVAL; file->f_pos = newpos; - dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos); + dbg_fops("%s: pos=%lld\n", __func__, (signed long long)newpos); #if 0 - pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n", + pr_err("0x%p, off %lld, whence %d -> pos %lld.\n", file, (signed long long)off, whence, (signed long long)off); #endif @@ -84,7 +179,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence) static int check_transfer_align(struct xdma_engine *engine, const char __user *buf, size_t count, loff_t pos, int sync) { - BUG_ON(!engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } /* AXI ST or AXI MM non-incremental addressing mode? */ if (engine->non_incr_addr) { @@ -175,17 +273,16 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) { struct sg_table *sgt = &cb->sgt; unsigned long len = cb->len; - char *buf = cb->buf; + void __user *buf = cb->buf; struct scatterlist *sg; - unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) - + unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE - 1) - ((unsigned long)buf & PAGE_MASK)) >> PAGE_SHIFT; int i; int rv; - if (pages_nr == 0) { + if (pages_nr == 0) return -EINVAL; - } if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) { pr_err("sgl OOM.\n"); @@ -211,8 +308,8 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) if (rv != pages_nr) { pr_err("unable to pin down all %u user pages, %d.\n", pages_nr, rv); - rv = -EFAULT; cb->pages_nr = rv; + rv = -EFAULT; goto err_out; } @@ -228,9 +325,9 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) sg = sgt->sgl; for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) { - //unsigned int offset = (uintptr_t)buf & ~PAGE_MASK; unsigned int offset = offset_in_page(buf); - unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len); + unsigned int nbytes = + min_t(unsigned int, PAGE_SIZE - offset, len); flush_dcache_page(cb->pages[i]); sg_set_page(sg, cb->pages[i], nbytes, offset); @@ -239,7 +336,10 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) len -= nbytes; } - BUG_ON(len); + if (len) { + pr_err("Invalid user buffer length. Cannot map to sgl\n"); + return -EINVAL; + } cb->pages_nr = pages_nr; return 0; @@ -249,7 +349,7 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) return rv; } -static ssize_t char_sgdma_read_write(struct file *file, char __user *buf, +static ssize_t char_sgdma_read_write(struct file *file, const char __user *buf, size_t count, loff_t *pos, bool write) { int rv; @@ -283,17 +383,16 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf, } memset(&cb, 0, sizeof(struct xdma_io_cb)); - cb.buf = buf; + cb.buf = (char __user *)buf; cb.len = count; + cb.ep_addr = (u64)*pos; + cb.write = write; rv = char_sgdma_map_user_buf_to_sgl(&cb, write); if (rv < 0) return rv; res = xdma_xfer_submit(xdev, engine->channel, write, *pos, &cb.sgt, - 0, sgdma_timeout * 1000); - //pr_err("xfer_submit return=%lld.\n", (s64)res); - - //interrupt_status(xdev); + 0, sgdma_timeout * 1000); char_sgdma_unmap_user_buf(&cb, write); @@ -302,151 +401,317 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf, static ssize_t char_sgdma_write(struct file *file, const char __user *buf, - size_t count, loff_t *pos) + size_t count, loff_t *pos) { - return char_sgdma_read_write(file, (char *)buf, count, pos, 1); + return char_sgdma_read_write(file, buf, count, pos, 1); } static ssize_t char_sgdma_read(struct file *file, char __user *buf, - size_t count, loff_t *pos) + size_t count, loff_t *pos) { - struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + return char_sgdma_read_write(file, buf, count, pos, 0); +} + +static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io, + unsigned long count, loff_t pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *) + iocb->ki_filp->private_data; + struct cdev_async_io *caio; struct xdma_engine *engine; + struct xdma_dev *xdev; int rv; + unsigned long i; - rv = xcdev_check(__func__, xcdev, 1); - if (rv < 0) - return rv; + if (!xcdev) { + pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n", + iocb->ki_filp, (u64)count, (u64)pos, 1); + return -EINVAL; + } engine = xcdev->engine; + xdev = xcdev->xdev; - if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { - rv = xdma_cyclic_transfer_setup(engine); - if (rv < 0 && rv != -EBUSY) + if (engine->dir != DMA_TO_DEVICE) { + pr_err("r/w mismatch. WRITE, dir %d.\n", + engine->dir); + return -EINVAL; + } + + caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL); + memset(caio, 0, sizeof(struct cdev_async_io)); + + caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL); + + spin_lock_init(&caio->lock); + iocb->private = caio; + caio->iocb = iocb; + caio->write = true; + caio->cancel = false; + caio->req_cnt = count; + + for (i = 0; i < count; i++) { + + memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb)); + + caio->cb[i].buf = io[i].iov_base; + caio->cb[i].len = io[i].iov_len; + caio->cb[i].ep_addr = (u64)pos; + caio->cb[i].write = true; + caio->cb[i].private = caio; + caio->cb[i].io_done = async_io_handler; + rv = check_transfer_align(engine, caio->cb[i].buf, + caio->cb[i].len, pos, 1); + if (rv) { + pr_info("Invalid transfer alignment detected\n"); + kmem_cache_free(cdev_cache, caio); + return rv; + } + + rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true); + if (rv < 0) return rv; - /* 600 sec. timeout */ - return xdma_engine_read_cyclic(engine, buf, count, 600000); + + rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, + engine->channel, caio->cb[i].write, + caio->cb[i].ep_addr, &caio->cb[i].sgt, + 0, sgdma_timeout * 1000); } - return char_sgdma_read_write(file, (char *)buf, count, pos, 0); + if (engine->cmplthp) + xdma_kthread_wakeup(engine->cmplthp); + + return -EIOCBQUEUED; } + +static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io, + unsigned long count, loff_t pos) +{ + + struct xdma_cdev *xcdev = (struct xdma_cdev *) + iocb->ki_filp->private_data; + struct cdev_async_io *caio; + struct xdma_engine *engine; + struct xdma_dev *xdev; + int rv; + unsigned long i; + + if (!xcdev) { + pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n", + iocb->ki_filp, (u64)count, (u64)pos, 1); + return -EINVAL; + } + + engine = xcdev->engine; + xdev = xcdev->xdev; + + if (engine->dir != DMA_FROM_DEVICE) { + pr_err("r/w mismatch. READ, dir %d.\n", + engine->dir); + return -EINVAL; + } + + caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL); + memset(caio, 0, sizeof(struct cdev_async_io)); + + caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL); + + spin_lock_init(&caio->lock); + iocb->private = caio; + caio->iocb = iocb; + caio->write = false; + caio->cancel = false; + caio->req_cnt = count; + + for (i = 0; i < count; i++) { + + memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb)); + + caio->cb[i].buf = io[i].iov_base; + caio->cb[i].len = io[i].iov_len; + caio->cb[i].ep_addr = (u64)pos; + caio->cb[i].write = false; + caio->cb[i].private = caio; + caio->cb[i].io_done = async_io_handler; + + rv = check_transfer_align(engine, caio->cb[i].buf, + caio->cb[i].len, pos, 1); + if (rv) { + pr_info("Invalid transfer alignment detected\n"); + kmem_cache_free(cdev_cache, caio); + return rv; + } + + rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true); + if (rv < 0) + return rv; + + rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, + engine->channel, caio->cb[i].write, + caio->cb[i].ep_addr, &caio->cb[i].sgt, + 0, sgdma_timeout * 1000); + } + + if (engine->cmplthp) + xdma_kthread_wakeup(engine->cmplthp); + + return -EIOCBQUEUED; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) +static ssize_t cdev_write_iter(struct kiocb *iocb, struct iov_iter *io) +{ + return cdev_aio_write(iocb, io->iov, io->nr_segs, io->iov_offset); +} + +static ssize_t cdev_read_iter(struct kiocb *iocb, struct iov_iter *io) +{ + return cdev_aio_read(iocb, io->iov, io->nr_segs, io->iov_offset); +} +#endif + static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg) { - int rv; - struct xdma_dev *xdev; - - BUG_ON(!engine); - xdev = engine->xdev; - BUG_ON(!xdev); - - /* performance measurement already running on this engine? */ - if (engine->xdma_perf) { - dbg_perf("IOCTL_XDMA_PERF_START failed!\n"); - dbg_perf("Perf measurement already seems to be running!\n"); - return -EBUSY; - } - engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), - GFP_KERNEL); - - if (!engine->xdma_perf) - return -ENOMEM; - - rv = copy_from_user(engine->xdma_perf, - (struct xdma_performance_ioctl *)arg, - sizeof(struct xdma_performance_ioctl)); - - if (rv < 0) { - dbg_perf("Failed to copy from user space 0x%lx\n", arg); - return -EINVAL; - } - if (engine->xdma_perf->version != IOCTL_XDMA_PERF_V1) { - dbg_perf("Unsupported IOCTL version %d\n", - engine->xdma_perf->version); - return -EINVAL; - } + int rv; + struct xdma_dev *xdev; - enable_perf(engine); - dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size); - /* initialize wait queue */ - init_waitqueue_head(&engine->xdma_perf_wq); - xdma_performance_submit(xdev, engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } + + xdev = engine->xdev; + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } + + /* performance measurement already running on this engine? */ + if (engine->xdma_perf) { + dbg_perf("IOCTL_XDMA_PERF_START failed!\n"); + dbg_perf("Perf measurement already seems to be running!\n"); + return -EBUSY; + } + engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), + GFP_KERNEL); - return 0; + if (!engine->xdma_perf) + return -ENOMEM; + + rv = copy_from_user(engine->xdma_perf, + (struct xdma_performance_ioctl __user *)arg, + sizeof(struct xdma_performance_ioctl)); + + if (rv < 0) { + dbg_perf("Failed to copy from user space 0x%lx\n", arg); + return -EINVAL; + } + if (engine->xdma_perf->version != IOCTL_XDMA_PERF_V1) { + dbg_perf("Unsupported IOCTL version %d\n", + engine->xdma_perf->version); + return -EINVAL; + } + + enable_perf(engine); + dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size); + /* initialize wait queue */ +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + init_swait_queue_head(&engine->xdma_perf_wq); +#else + init_waitqueue_head(&engine->xdma_perf_wq); +#endif + rv = xdma_performance_submit(xdev, engine); + if (rv < 0) + pr_err("Failed to submit dma performance\n"); + return rv; } static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg) { - struct xdma_transfer *transfer = NULL; - int rv; - - dbg_perf("IOCTL_XDMA_PERF_STOP\n"); - - /* no performance measurement running on this engine? */ - if (!engine->xdma_perf) { - dbg_perf("No measurement in progress\n"); - return -EINVAL; - } - - /* stop measurement */ - transfer = engine_cyclic_stop(engine); - dbg_perf("Waiting for measurement to stop\n"); - - if (engine->xdma_perf) { - get_perf_stats(engine); - - rv = copy_to_user((void __user *)arg, engine->xdma_perf, - sizeof(struct xdma_performance_ioctl)); - if (rv) { - dbg_perf("Error copying result to user\n"); - return -EINVAL; - } - } else { - dbg_perf("engine->xdma_perf == NULL?\n"); - } - - kfree(engine->xdma_perf); - engine->xdma_perf = NULL; - - return 0; + struct xdma_transfer *transfer = NULL; + int rv; + + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } + + dbg_perf("IOCTL_XDMA_PERF_STOP\n"); + + /* no performance measurement running on this engine? */ + if (!engine->xdma_perf) { + dbg_perf("No measurement in progress\n"); + return -EINVAL; + } + + /* stop measurement */ + transfer = engine_cyclic_stop(engine); + if (!transfer) { + pr_err("Failed to stop cyclic transfer\n"); + return -EINVAL; + } + dbg_perf("Waiting for measurement to stop\n"); + + get_perf_stats(engine); + + rv = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rv) { + dbg_perf("Error copying result to user\n"); + return rv; + } + + kfree(transfer); + + kfree(engine->xdma_perf); + engine->xdma_perf = NULL; + + return 0; } static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg) { - int rc; + int rc; - BUG_ON(!engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } - dbg_perf("IOCTL_XDMA_PERF_GET\n"); + dbg_perf("IOCTL_XDMA_PERF_GET\n"); - if (engine->xdma_perf) { - get_perf_stats(engine); + if (engine->xdma_perf) { + get_perf_stats(engine); - rc = copy_to_user((void __user *)arg, engine->xdma_perf, - sizeof(struct xdma_performance_ioctl)); - if (rc) { - dbg_perf("Error copying result to user\n"); - return -EINVAL; - } - } else { - dbg_perf("engine->xdma_perf == NULL?\n"); - return -EPROTO; - } + rc = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rc) { + dbg_perf("Error copying result to user\n"); + return rc; + } + } else { + dbg_perf("engine->xdma_perf == NULL?\n"); + return -EPROTO; + } - return 0; + return 0; } -static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) +static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) { return engine_addrmode_set(engine, arg); } -static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) +static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) { int rv; unsigned long src; - BUG_ON(!engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } src = !!engine->non_incr_addr; dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n"); @@ -455,22 +720,25 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) return rv; } -static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) +static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) { - BUG_ON(!engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return -EINVAL; + } dbg_perf("IOCTL_XDMA_ALIGN_GET\n"); return put_user(engine->addr_align, (int __user *)arg); } static long char_sgdma_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) + unsigned long arg) { struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; struct xdma_dev *xdev; struct xdma_engine *engine; - int rv = 0; + int rv = 0; rv = xcdev_check(__func__, xcdev, 1); if (rv < 0) @@ -479,16 +747,16 @@ static long char_sgdma_ioctl(struct file *file, unsigned int cmd, xdev = xcdev->xdev; engine = xcdev->engine; - switch (cmd) { - case IOCTL_XDMA_PERF_START: - rv = ioctl_do_perf_start(engine, arg); - break; - case IOCTL_XDMA_PERF_STOP: - rv = ioctl_do_perf_stop(engine, arg); - break; - case IOCTL_XDMA_PERF_GET: - rv = ioctl_do_perf_get(engine, arg); - break; + switch (cmd) { + case IOCTL_XDMA_PERF_START: + rv = ioctl_do_perf_start(engine, arg); + break; + case IOCTL_XDMA_PERF_STOP: + rv = ioctl_do_perf_stop(engine, arg); + break; + case IOCTL_XDMA_PERF_GET: + rv = ioctl_do_perf_get(engine, arg); + break; case IOCTL_XDMA_ADDRMODE_SET: rv = ioctl_do_addrmode_set(engine, arg); break; @@ -498,13 +766,13 @@ static long char_sgdma_ioctl(struct file *file, unsigned int cmd, case IOCTL_XDMA_ALIGN_GET: rv = ioctl_do_align_get(engine, arg); break; - default: - dbg_perf("Unsupported operation\n"); - rv = -EINVAL; - break; - } + default: + dbg_perf("Unsupported operation\n"); + rv = -EINVAL; + break; + } - return rv; + return rv; } static int char_sgdma_open(struct inode *inode, struct file *file) @@ -520,8 +788,7 @@ static int char_sgdma_open(struct inode *inode, struct file *file) if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { if (engine->device_open == 1) return -EBUSY; - else - engine->device_open = 1; + engine->device_open = 1; } return 0; @@ -552,7 +819,17 @@ static const struct file_operations sgdma_fops = { .open = char_sgdma_open, .release = char_sgdma_close, .write = char_sgdma_write, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) + .write_iter = cdev_write_iter, +#else + .aio_write = cdev_aio_write, +#endif .read = char_sgdma_read, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) + .read_iter = cdev_read_iter, +#else + .aio_read = cdev_aio_read, +#endif .unlocked_ioctl = char_sgdma_ioctl, .llseek = char_sgdma_llseek, }; diff --git a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.h b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.h index c67bf99f5..4f0a38cce 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.h +++ b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef _XDMA_IOCALLS_POSIX_H_ #define _XDMA_IOCALLS_POSIX_H_ @@ -50,17 +51,16 @@ * _IOC_SIZE(nr) returns size */ -struct xdma_performance_ioctl -{ - /* IOCTL_XDMA_IOCTL_Vx */ - uint32_t version; - uint32_t transfer_size; - /* measurement */ - uint32_t stopped; - uint32_t iterations; - uint64_t clock_cycle_count; - uint64_t data_cycle_count; - uint64_t pending_count; +struct xdma_performance_ioctl { + /* IOCTL_XDMA_IOCTL_Vx */ + uint32_t version; + uint32_t transfer_size; + /* measurement */ + uint32_t stopped; + uint32_t iterations; + uint64_t clock_cycle_count; + uint64_t data_cycle_count; + uint64_t pending_count; }; diff --git a/sdk/linux_kernel_drivers/xdma/cdev_xvc.c b/sdk/linux_kernel_drivers/xdma/cdev_xvc.c index adafa7fc8..b2ea63484 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_xvc.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_xvc.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include "xdma_cdev.h" @@ -36,30 +37,30 @@ #ifdef __REG_DEBUG__ /* SECTION: Function definitions */ -inline void __write_register(const char *fn, u32 value, void *base, +inline void __write_register(const char *fn, u32 value, void __iomem *base, unsigned int off) { - pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value); - iowrite32(value, base + off); + pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value); + iowrite32(value, base + off); } -inline u32 __read_register(const char *fn, void *base, unsigned int off) +inline u32 __read_register(const char *fn, void __iomem *base, unsigned int off) { u32 v = ioread32(base + off); - pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v); - return v; + pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v); + return v; } -#define write_register(v,base,off) __write_register(__func__, v, base, off) -#define read_register(base,off) __read_register(__func__, base, off) +#define write_register(v, base, off) __write_register(__func__, v, base, off) +#define read_register(base, off) __read_register(__func__, base, off) #else -#define write_register(v,base,off) iowrite32(v, (base) + (off)) -#define read_register(base,off) ioread32((base) + (off)) +#define write_register(v, base, off) iowrite32(v, (base) + (off)) +#define read_register(base, off) ioread32((base) + (off)) #endif /* #ifdef __REG_DEBUG__ */ -static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits, +static int xvc_shift_bits(void __iomem *base, u32 tms_bits, u32 tdi_bits, u32 *tdo_bits) { u32 control; @@ -96,7 +97,7 @@ static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits, static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { - struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; + struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; struct xdma_dev *xdev; struct xvc_ioc xvc_obj; unsigned int opcode; @@ -113,6 +114,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) rv = xcdev_check(__func__, xcdev, 0); if (rv < 0) return rv; + xdev = xcdev->xdev; if (cmd != XDMA_IOCXVC) { @@ -139,7 +141,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) total_bits = xvc_obj.length; total_bytes = (total_bits + 7) >> 3; - buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL); + buffer = kmalloc(total_bytes * 3, GFP_KERNEL); if (!buffer) { pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n", 3 * total_bytes, opcode, total_bits, total_bytes); @@ -150,12 +152,16 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) tdi_buf = tms_buf + total_bytes; tdo_buf = tdi_buf + total_bytes; - rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes); + rv = copy_from_user((void *)tms_buf, + (const char __user *)xvc_obj.tms_buf, + total_bytes); if (rv) { pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes); goto cleanup; } - rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes); + rv = copy_from_user((void *)tdi_buf, + (const char __user *)xvc_obj.tdi_buf, + total_bytes); if (rv) { pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes); goto cleanup; @@ -166,7 +172,8 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) iobase = xdev->bar[xcdev->bar] + xcdev->base; /* set length register to 32 initially if more than one - * word-transaction is to be done */ + * word-transaction is to be done + */ if (total_bits >= 32) write_register(0x20, iobase, XVC_BAR_LENGTH_REG); @@ -177,7 +184,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) u32 tms_store = 0; u32 tdi_store = 0; u32 tdo_store = 0; - + if (bits_left < 32) { /* set number of bits to shift out */ write_register(bits_left, iobase, XVC_BAR_LENGTH_REG); @@ -190,33 +197,35 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) /* Shift data out and copy to output buffer */ rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store); if (rv < 0) - goto cleanup; + break; memcpy(tdo_buf + bytes, &tdo_store, shift_bytes); } + if (rv < 0) + goto unlock; + /* if testing bar access swap tdi and tdo bufferes to "loopback" */ if (opcode == 0x2) { - char *tmp = tdo_buf; + unsigned char *tmp = tdo_buf; tdo_buf = tdi_buf; tdi_buf = tmp; } - rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes); - if (rv) { + rv = copy_to_user(xvc_obj.tdo_buf, (const void *)tdo_buf, total_bytes); + if (rv) pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes); - rv = -EFAULT; - goto cleanup; - } - -cleanup: - if (buffer) - kfree(buffer); +unlock: +#if KERNEL_VERSION(5, 1, 0) >= LINUX_VERSION_CODE mmiowb(); +#endif spin_unlock(&xcdev->lock); +cleanup: + kfree(buffer); + return rv; } @@ -224,10 +233,10 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) * character device file operations for the XVC */ static const struct file_operations xvc_fops = { - .owner = THIS_MODULE, - .open = char_open, - .release = char_close, - .unlocked_ioctl = xvc_ioctl, + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .unlocked_ioctl = xvc_ioctl, }; void cdev_xvc_init(struct xdma_cdev *xcdev) diff --git a/sdk/linux_kernel_drivers/xdma/cdev_xvc.h b/sdk/linux_kernel_drivers/xdma/cdev_xvc.h index de9473a37..8e0a4bec1 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_xvc.h +++ b/sdk/linux_kernel_drivers/xdma/cdev_xvc.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef __XVC_IOCTL_H__ #define __XVC_IOCTL_H__ @@ -37,9 +38,9 @@ struct xvc_ioc { unsigned int opcode; unsigned int length; - unsigned char *tms_buf; - unsigned char *tdi_buf; - unsigned char *tdo_buf; + const char __user *tms_buf; + const char __user *tdi_buf; + void __user *tdo_buf; }; #define XDMA_IOCXVC _IOWR(XVC_MAGIC, 1, struct xvc_ioc) diff --git a/sdk/linux_kernel_drivers/xdma/libxdma.c b/sdk/linux_kernel_drivers/xdma/libxdma.c index 32523e500..d91bcf724 100755 --- a/sdk/linux_kernel_drivers/xdma/libxdma.c +++ b/sdk/linux_kernel_drivers/xdma/libxdma.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -22,7 +22,7 @@ * ******************************************************************************/ -#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include #include @@ -35,17 +35,17 @@ #include "libxdma.h" #include "libxdma_api.h" #include "cdev_sgdma.h" +#include "xdma_thread.h" /* SECTION: Module licensing */ #ifdef __LIBXDMA_MOD__ #include "version.h" -#define DRV_MODULE_NAME "libxdma" -#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" -#define DRV_MODULE_RELDATE "Feb. 2017" +#define DRV_MODULE_NAME "libxdma" +#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" static char version[] = - DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; MODULE_AUTHOR("Xilinx, Inc."); MODULE_DESCRIPTION(DRV_MODULE_DESC); @@ -53,6 +53,8 @@ MODULE_VERSION(DRV_MODULE_VERSION); MODULE_LICENSE("GPL v2"); #endif +extern unsigned int desc_blen_max; + /* Module Parameters */ static unsigned int poll_mode; module_param(poll_mode, uint, 0644); @@ -62,13 +64,38 @@ static unsigned int interrupt_mode; module_param(interrupt_mode, uint, 0644); MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); -static unsigned int enable_credit_mp; +static unsigned int enable_credit_mp = 1; module_param(enable_credit_mp, uint, 0644); -MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); +MODULE_PARM_DESC( + enable_credit_mp, + "Set 0 to disable credit feature, default is 1 ( credit control enabled)"); unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; module_param(desc_blen_max, uint, 0644); -MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); +MODULE_PARM_DESC(desc_blen_max, + "per descriptor max. buffer length, default is (1 << 28) - 1"); + +#define XDMA_PERF_NUM_DESC 128 + +/* Kernel version adaptative code */ +#if KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE +/* since 4.18, using simple wait queues is not recommended + * except for realtime constraint (see swait.h comments) + * and will likely be removed in future kernel versions + */ +#define xlx_wake_up swake_up_one +#define xlx_wait_event_interruptible_timeout \ + swait_event_interruptible_timeout_exclusive +#elif KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE +#define xlx_wake_up swake_up +#define xlx_wait_event_interruptible_timeout \ + swait_event_interruptible_timeout +#else +#define xlx_wake_up wake_up_interruptible +#define xlx_wait_event_interruptible_timeout \ + wait_event_interruptible_timeout +#endif + /* * xdma device management @@ -81,8 +108,7 @@ static LIST_HEAD(xdev_rcu_list); static DEFINE_SPINLOCK(xdev_rcu_lock); #ifndef list_last_entry -#define list_last_entry(ptr, type, member) \ - list_entry((ptr)->prev, type, member) +#define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) #endif static inline void xdev_list_add(struct xdma_dev *xdev) @@ -100,7 +126,7 @@ static inline void xdev_list_add(struct xdma_dev *xdev) mutex_unlock(&xdev_mutex); dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", - dev_name(&xdev->pdev->dev), xdev, xdev->idx); + dev_name(&xdev->pdev->dev), xdev, xdev->idx); spin_lock(&xdev_rcu_lock); list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); @@ -123,22 +149,22 @@ static inline void xdev_list_remove(struct xdma_dev *xdev) struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) { - struct xdma_dev *xdev, *tmp; + struct xdma_dev *xdev, *tmp; - mutex_lock(&xdev_mutex); - list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { - if (xdev->pdev == pdev) { - mutex_unlock(&xdev_mutex); - return xdev; - } - } - mutex_unlock(&xdev_mutex); - return NULL; + mutex_lock(&xdev_mutex); + list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { + if (xdev->pdev == pdev) { + mutex_unlock(&xdev_mutex); + return xdev; + } + } + mutex_unlock(&xdev_mutex); + return NULL; } EXPORT_SYMBOL_GPL(xdev_find_by_pdev); static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, - void *hndl) + void *hndl) { struct xdma_dev *xdev; @@ -147,13 +173,13 @@ static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, xdev = xdev_find_by_pdev(pdev); if (!xdev) { - pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", - fname, pdev, hndl); + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", fname, + pdev, hndl); return -EINVAL; } if (xdev != hndl) { - pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", - fname, pdev, hndl, xdev); + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", fname, pdev, hndl, + xdev); return -EINVAL; } @@ -162,14 +188,15 @@ static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, #ifdef __LIBXDMA_DEBUG__ /* SECTION: Function definitions */ -inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) +inline void __write_register(const char *fn, u32 value, void *iomem, + unsigned long off) { pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); iowrite32(value, iomem); } -#define write_register(v,mem,off) __write_register(__func__, v, mem, off) +#define write_register(v, mem, off) __write_register(__func__, v, mem, off) #else -#define write_register(v,mem,off) iowrite32(v, mem) +#define write_register(v, mem, off) iowrite32(v, mem) #endif inline u32 read_register(void *iomem) @@ -189,8 +216,9 @@ static inline u64 build_u64(u64 hi, u64 lo) static void check_nonzero_interrupt_status(struct xdma_dev *xdev) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 w; w = read_register(®->user_int_enable); @@ -200,7 +228,7 @@ static void check_nonzero_interrupt_status(struct xdma_dev *xdev) w = read_register(®->channel_int_enable); if (w) - pr_info("%s xdma%d channel_int_enable = 0x%08x\n", + pr_info("%s xdma%d channel_int_enable = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(®->user_int_request); @@ -225,8 +253,9 @@ static void check_nonzero_interrupt_status(struct xdma_dev *xdev) /* channel_interrupts_enable -- Enable interrupts we are interested in */ static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); } @@ -234,8 +263,9 @@ static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) /* channel_interrupts_disable -- Disable interrupts we not interested in */ static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); } @@ -243,8 +273,9 @@ static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) /* user_interrupts_enable -- Enable interrupts we are interested in */ static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); } @@ -252,8 +283,9 @@ static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) /* user_interrupts_disable -- Disable interrupts we not interested in */ static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); } @@ -261,18 +293,19 @@ static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) /* read_interrupts -- Print the interrupt controller status */ static u32 read_interrupts(struct xdma_dev *xdev) { - struct interrupt_regs *reg = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 lo; u32 hi; /* extra debugging; inspect complete engine set of registers */ hi = read_register(®->user_int_request); dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", - ®->user_int_request, hi); + ®->user_int_request, hi); lo = read_register(®->channel_int_request); dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", - ®->channel_int_request, lo); + ®->channel_int_request, lo); /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ return build_u32(hi, lo); @@ -284,17 +317,16 @@ void enable_perf(struct xdma_engine *engine) w = XDMA_PERF_CLEAR; write_register(w, &engine->regs->perf_ctrl, - (unsigned long)(&engine->regs->perf_ctrl) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); read_register(&engine->regs->identifier); w = XDMA_PERF_AUTO | XDMA_PERF_RUN; write_register(w, &engine->regs->perf_ctrl, - (unsigned long)(&engine->regs->perf_ctrl) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); read_register(&engine->regs->identifier); dbg_perf("IOCTL_XDMA_PERF_START\n"); - } EXPORT_SYMBOL_GPL(enable_perf); @@ -303,7 +335,10 @@ void get_perf_stats(struct xdma_engine *engine) u32 hi; u32 lo; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return; + } if (!engine->xdma_perf) { pr_info("%s perf struct not set up.\n", engine->name); @@ -329,34 +364,37 @@ void get_perf_stats(struct xdma_engine *engine) } EXPORT_SYMBOL_GPL(get_perf_stats); -static void engine_reg_dump(struct xdma_engine *engine) +static int engine_reg_dump(struct xdma_engine *engine) { u32 w; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } w = read_register(&engine->regs->identifier); - pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", - engine->name, &engine->regs->identifier, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", engine->name, + &engine->regs->identifier, w); w &= BLOCK_ID_MASK; if (w != BLOCK_ID_HEAD) { - pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", - engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); - return; + pr_err("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return -EINVAL; } /* extra debugging; inspect complete engine set of registers */ w = read_register(&engine->regs->status); - pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", - engine->name, &engine->regs->status, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", engine->name, + &engine->regs->status, w); w = read_register(&engine->regs->control); - pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", - engine->name, &engine->regs->control, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", engine->name, + &engine->regs->control, w); w = read_register(&engine->sgdma_regs->first_desc_lo); - pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", - engine->name, &engine->sgdma_regs->first_desc_lo, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", engine->name, + &engine->sgdma_regs->first_desc_lo, w); w = read_register(&engine->sgdma_regs->first_desc_hi); - pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", - engine->name, &engine->sgdma_regs->first_desc_hi, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", engine->name, + &engine->sgdma_regs->first_desc_hi, w); w = read_register(&engine->sgdma_regs->first_desc_adjacent); pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", engine->name, &engine->sgdma_regs->first_desc_adjacent, w); @@ -366,15 +404,10 @@ static void engine_reg_dump(struct xdma_engine *engine) w = read_register(&engine->regs->interrupt_enable_mask); pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", engine->name, &engine->regs->interrupt_enable_mask, w); + + return 0; } -/** - * engine_status_read() - read status of SG DMA engine (optionally reset) - * - * Stores status in engine->status. - * - * @return -1 on failure, status register otherwise - */ static void engine_status_dump(struct xdma_engine *engine) { u32 v = engine->status; @@ -383,7 +416,7 @@ static void engine_status_dump(struct xdma_engine *engine) int len = 0; len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); - + if ((v & XDMA_STAT_BUSY)) len += sprintf(buf + len, "BUSY,"); if ((v & XDMA_STAT_DESC_STOPPED)) @@ -391,8 +424,8 @@ static void engine_status_dump(struct xdma_engine *engine) if ((v & XDMA_STAT_DESC_COMPLETED)) len += sprintf(buf + len, "DESC_COMPL,"); - /* common H2C & C2H */ - if ((v & XDMA_STAT_COMMON_ERR_MASK)) { + /* common H2C & C2H */ + if ((v & XDMA_STAT_COMMON_ERR_MASK)) { if ((v & XDMA_STAT_ALIGN_MISMATCH)) len += sprintf(buf + len, "ALIGN_MISMATCH "); if ((v & XDMA_STAT_MAGIC_STOPPED)) @@ -404,7 +437,7 @@ static void engine_status_dump(struct xdma_engine *engine) buf[len - 1] = ','; } - if ((engine->dir == DMA_TO_DEVICE)) { + if (engine->dir == DMA_TO_DEVICE) { /* H2C only */ if ((v & XDMA_STAT_H2C_R_ERR_MASK)) { len += sprintf(buf + len, "R:"); @@ -429,7 +462,7 @@ static void engine_status_dump(struct xdma_engine *engine) len += sprintf(buf + len, "SLAVE_ERR "); buf[len - 1] = ','; } - + } else { /* C2H only */ if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { @@ -442,8 +475,8 @@ static void engine_status_dump(struct xdma_engine *engine) } } - /* common H2C & C2H */ - if ((v & XDMA_STAT_DESC_ERR_MASK)) { + /* common H2C & C2H */ + if ((v & XDMA_STAT_DESC_ERR_MASK)) { len += sprintf(buf + len, "DESC_ERR:"); if ((v & XDMA_STAT_DESC_UNSUPP_REQ)) len += sprintf(buf + len, "UNSUPP_REQ "); @@ -462,38 +495,55 @@ static void engine_status_dump(struct xdma_engine *engine) pr_info("%s\n", buffer); } -static u32 engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return error value on failure, 0 otherwise + */ +static int engine_status_read(struct xdma_engine *engine, bool clear, bool dump) { - u32 value; + int rv = 0; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } - if (dump) - engine_reg_dump(engine); + if (dump) { + rv = engine_reg_dump(engine); + if (rv < 0) { + pr_err("Failed to dump register\n"); + return rv; + } + } /* read status register */ if (clear) - value = engine->status = - read_register(&engine->regs->status_rc); + engine->status = read_register(&engine->regs->status_rc); else - value = engine->status = read_register(&engine->regs->status); + engine->status = read_register(&engine->regs->status); if (dump) engine_status_dump(engine); - return value; + return rv; } /** * xdma_engine_stop() - stop an SG DMA engine * */ -static void xdma_engine_stop(struct xdma_engine *engine) +static int xdma_engine_stop(struct xdma_engine *engine) { u32 w; - BUG_ON(!engine); - dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + dbg_tfr("%s(engine=%p)\n", __func__, engine); w = 0; w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; @@ -502,31 +552,30 @@ static void xdma_engine_stop(struct xdma_engine *engine) w |= (u32)XDMA_CTRL_IE_DESC_ERROR; if (poll_mode) { - w |= (u32) XDMA_CTRL_POLL_MODE_WB; + w |= (u32)XDMA_CTRL_POLL_MODE_WB; } else { w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; - - /* Disable IDLE STOPPED for MM */ - if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || - (engine->xdma_perf)) - w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; } dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", - engine->name, w, (u32 *)&engine->regs->control); + engine->name, w, (u32 *)&engine->regs->control); write_register(w, &engine->regs->control, (unsigned long)(&engine->regs->control) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); /* dummy read of status register to flush all previous writes */ - dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); + dbg_tfr("%s(%s) done\n", __func__, engine->name); + return 0; } -static void engine_start_mode_config(struct xdma_engine *engine) +static int engine_start_mode_config(struct xdma_engine *engine) { u32 w; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } /* If a perf test is running, enable the engine interrupts */ if (engine->xdma_perf) { @@ -538,9 +587,10 @@ static void engine_start_mode_config(struct xdma_engine *engine) w |= XDMA_CTRL_IE_READ_ERROR; w |= XDMA_CTRL_IE_DESC_ERROR; - write_register(w, &engine->regs->interrupt_enable_mask, + write_register( + w, &engine->regs->interrupt_enable_mask, (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); } /* write control register of SG DMA engine */ @@ -552,30 +602,28 @@ static void engine_start_mode_config(struct xdma_engine *engine) if (poll_mode) { w |= (u32)XDMA_CTRL_POLL_MODE_WB; - } else { + } else { w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; - - if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || - (engine->xdma_perf)) - w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; - - /* set non-incremental addressing mode */ - if (engine->non_incr_addr) - w |= (u32)XDMA_CTRL_NON_INCR_ADDR; } + /* set non-incremental addressing mode */ + if (engine->non_incr_addr) + w |= (u32)XDMA_CTRL_NON_INCR_ADDR; + dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, - (void *)&engine->regs->control); + (void *)&engine->regs->control); + /* start the engine */ write_register(w, &engine->regs->control, - (unsigned long)(&engine->regs->control) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); /* dummy read of status register to flush all previous writes */ w = read_register(&engine->regs->status); dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", - &engine->regs->status, w); + &engine->regs->status, w); + return 0; } /** @@ -599,20 +647,46 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) struct xdma_transfer *transfer; u32 w; int extra_adj = 0; + int rv; + + if (!engine) { + pr_err("dma engine NULL\n"); + return NULL; + } /* engine must be idle */ - BUG_ON(engine->running); + if (engine->running) { + pr_info("%s engine is not in idle state to start\n", + engine->name); + return NULL; + } + /* engine transfer queue must not be empty */ - BUG_ON(list_empty(&engine->transfer_list)); + if (list_empty(&engine->transfer_list)) { + pr_debug("%s engine transfer queue must not be empty\n", + engine->name); + return NULL; + } /* inspect first transfer queued on the engine */ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); - BUG_ON(!transfer); + entry); + if (!transfer) { + pr_debug("%s queued transfer must not be empty\n", + engine->name); + return NULL; + } /* engine is no longer shutdown */ engine->shutdown = ENGINE_SHUTDOWN_NONE; - dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); + dbg_tfr("%s(%s): transfer=0x%p.\n", __func__, engine->name, transfer); + + /* Add credits for Streaming mode C2H */ + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + if (enable_credit_mp) + write_register(engine->desc_used, + &engine->sgdma_regs->credits, 0); + } /* initialize number of descriptors of dequeued transfers */ engine->desc_dequeued = 0; @@ -620,37 +694,48 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) /* write lower 32-bit of bus address of transfer first descriptor */ w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, - (void *)&engine->sgdma_regs->first_desc_lo); + (void *)&engine->sgdma_regs->first_desc_lo); write_register(w, &engine->sgdma_regs->first_desc_lo, - (unsigned long)(&engine->sgdma_regs->first_desc_lo) - - (unsigned long)(&engine->sgdma_regs)); + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); /* write upper 32-bit of bus address of transfer first descriptor */ w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, - (void *)&engine->sgdma_regs->first_desc_hi); + (void *)&engine->sgdma_regs->first_desc_hi); write_register(w, &engine->sgdma_regs->first_desc_hi, - (unsigned long)(&engine->sgdma_regs->first_desc_hi) - - (unsigned long)(&engine->sgdma_regs)); + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); if (transfer->desc_adjacent > 0) { extra_adj = transfer->desc_adjacent - 1; if (extra_adj > MAX_EXTRA_ADJ) extra_adj = MAX_EXTRA_ADJ; } - dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", - extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); - write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, - (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", extra_adj, + (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register( + extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - (unsigned long)(&engine->sgdma_regs)); dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", &engine->regs->status); - mmiowb(); - engine_start_mode_config(engine); +#if KERNEL_VERSION(5, 1, 0) >= LINUX_VERSION_CODE + mmiowb(); +#endif - engine_status_read(engine, 0, 0); + rv = engine_start_mode_config(engine); + if (rv < 0) { + pr_err("Failed to start engine mode config\n"); + return NULL; + } + rv = engine_status_read(engine, 0, 0); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + return NULL; + } dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); /* remember the engine is running */ engine->running = 1; @@ -665,21 +750,31 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) * @engine pointer to struct xdma_engine * */ -static void engine_service_shutdown(struct xdma_engine *engine) +static int engine_service_shutdown(struct xdma_engine *engine) { + int rv; /* if the engine stopped with RUN still asserted, de-assert RUN now */ dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); - xdma_engine_stop(engine); + rv = xdma_engine_stop(engine); + if (rv < 0) { + pr_err("Failed to stop engine\n"); + return rv; + } engine->running = 0; /* awake task on engine's shutdown wait queue */ - wake_up_interruptible(&engine->shutdown_wq); + xlx_wake_up(&engine->shutdown_wq); + return 0; } -struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, +static struct xdma_transfer *engine_transfer_completion( + struct xdma_engine *engine, struct xdma_transfer *transfer) { - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return NULL; + } if (unlikely(!transfer)) { pr_info("%s: xfer empty.\n", engine->name); @@ -688,20 +783,33 @@ struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, /* synchronous I/O? */ /* awake task on transfer's wait queue */ - wake_up_interruptible(&transfer->wq); + xlx_wake_up(&transfer->wq); + + /* Send completion notification for Last transfer */ + if (transfer->cb && transfer->last_in_request) + transfer->cb->io_done((unsigned long)transfer->cb, 0); return transfer; } -struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, - struct xdma_transfer *transfer, u32 *pdesc_completed) +static struct xdma_transfer * +engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, + u32 *pdesc_completed) { - BUG_ON(!engine); - BUG_ON(!pdesc_completed); + if (!engine) { + pr_err("dma engine NULL\n"); + return NULL; + } + + if (!pdesc_completed) { + pr_err("%s completed descriptors are null.\n", engine->name); + return NULL; + } if (unlikely(!transfer)) { - pr_info("%s xfer empty, pdesc completed %u.\n", - engine->name, *pdesc_completed); + pr_info("%s xfer empty, pdesc completed %u.\n", engine->name, + *pdesc_completed); return NULL; } @@ -710,11 +818,12 @@ struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, * except for the last (i.e. use > instead of >=). */ while (transfer && (!transfer->cyclic) && - (*pdesc_completed > transfer->desc_num)) { + (*pdesc_completed > transfer->desc_num)) { /* remove this transfer from pdesc_completed */ *pdesc_completed -= transfer->desc_num; dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ list_del(engine->transfer_list.next); /* add to dequeued number of descriptors during this run */ @@ -722,14 +831,16 @@ struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, /* mark transfer as succesfully completed */ transfer->state = TRANSFER_STATE_COMPLETED; - /* Complete transfer - sets transfer to NULL if an async - * transfer has completed */ + /* + * Complete transfer - sets transfer to NULL if an async + * transfer has completed + */ transfer = engine_transfer_completion(engine, transfer); /* if exists, get the next transfer on the list */ if (!list_empty(&engine->transfer_list)) { transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); + struct xdma_transfer, entry); dbg_tfr("Non-completed transfer %p\n", transfer); } else { /* no further transfers? */ @@ -740,11 +851,11 @@ struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, return transfer; } -static void engine_err_handle(struct xdma_engine *engine, - struct xdma_transfer *transfer, u32 desc_completed) +static int engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) { u32 value; - + int rv = 0; /* * The BUSY bit is expected to be clear now but older HW has a race * condition which could cause it to be still set. If it's set, re-read @@ -752,91 +863,103 @@ static void engine_err_handle(struct xdma_engine *engine, */ if (engine->status & XDMA_STAT_BUSY) { value = read_register(&engine->regs->status); - if ((value & XDMA_STAT_BUSY) && printk_ratelimit()) - pr_info("%s has errors but is still BUSY\n", + if ((value & XDMA_STAT_BUSY)) + printk_ratelimited(KERN_INFO "%s has errors but is still BUSY\n", engine->name); } - if (printk_ratelimit()) { - pr_info("%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + printk_ratelimited(KERN_INFO "%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", engine->name, engine->status, transfer, desc_completed, transfer->desc_num); - } - + /* mark transfer as failed */ transfer->state = TRANSFER_STATE_FAILED; - xdma_engine_stop(engine); + rv = xdma_engine_stop(engine); + if (rv < 0) + pr_err("Failed to stop engine\n"); + return rv; } -struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, - struct xdma_transfer *transfer, u32 *pdesc_completed) +static struct xdma_transfer * +engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, + u32 *pdesc_completed) { - BUG_ON(!engine); - BUG_ON(!pdesc_completed); + if (!engine) { + pr_err("dma engine NULL\n"); + return NULL; + } + + if (!pdesc_completed) { + pr_err("%s completed descriptors are null.\n", engine->name); + return NULL; + } /* inspect the current transfer */ if (unlikely(!transfer)) { - pr_info("%s xfer empty, pdesc completed %u.\n", - engine->name, *pdesc_completed); + pr_info("%s xfer empty, pdesc completed %u.\n", engine->name, + *pdesc_completed); return NULL; - } else { - if (((engine->dir == DMA_FROM_DEVICE) && - (engine->status & XDMA_STAT_C2H_ERR_MASK)) || - ((engine->dir == DMA_TO_DEVICE) && - (engine->status & XDMA_STAT_H2C_ERR_MASK))) { - pr_info("engine %s, status error 0x%x.\n", - engine->name, engine->status); - engine_status_dump(engine); - engine_err_handle(engine, transfer, *pdesc_completed); - goto transfer_del; - } + } + if (((engine->dir == DMA_FROM_DEVICE) && + (engine->status & XDMA_STAT_C2H_ERR_MASK)) || + ((engine->dir == DMA_TO_DEVICE) && + (engine->status & XDMA_STAT_H2C_ERR_MASK))) { + pr_info("engine %s, status error 0x%x.\n", engine->name, + engine->status); + engine_status_dump(engine); + engine_err_handle(engine, transfer, *pdesc_completed); + goto transfer_del; + } - if (engine->status & XDMA_STAT_BUSY) - pr_debug("engine %s is unexpectedly busy - ignoring\n", - engine->name); + if (engine->status & XDMA_STAT_BUSY) + pr_debug("engine %s is unexpectedly busy - ignoring\n", + engine->name); - /* the engine stopped on current transfer? */ - if (*pdesc_completed < transfer->desc_num) { - transfer->state = TRANSFER_STATE_FAILED; - pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", - engine->name, transfer, *pdesc_completed, - transfer->desc_num); - } else { - dbg_tfr("engine %s completed transfer\n", engine->name); - dbg_tfr("Completed transfer ID = 0x%p\n", transfer); - dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", - *pdesc_completed, transfer->desc_num); - - if (!transfer->cyclic) { - /* - * if the engine stopped on this transfer, - * it should be the last - */ - WARN_ON(*pdesc_completed > transfer->desc_num); - } - /* mark transfer as succesfully completed */ - transfer->state = TRANSFER_STATE_COMPLETED; + /* the engine stopped on current transfer? */ + if (*pdesc_completed < transfer->desc_num) { + transfer->state = TRANSFER_STATE_FAILED; + pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", + engine->name, transfer, *pdesc_completed, + transfer->desc_num); + } else { + dbg_tfr("engine %s completed transfer\n", engine->name); + dbg_tfr("Completed transfer ID = 0x%p\n", transfer); + dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", + *pdesc_completed, transfer->desc_num); + + if (!transfer->cyclic) { + /* + * if the engine stopped on this transfer, + * it should be the last + */ + WARN_ON(*pdesc_completed > transfer->desc_num); } + /* mark transfer as successfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + } transfer_del: - /* remove completed transfer from list */ - list_del(engine->transfer_list.next); - /* add to dequeued number of descriptors during this run */ - engine->desc_dequeued += transfer->desc_num; + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; - /* - * Complete transfer - sets transfer to NULL if an asynchronous - * transfer has completed - */ - transfer = engine_transfer_completion(engine, transfer); - } + /* + * Complete transfer - sets transfer to NULL if an asynchronous + * transfer has completed + */ + transfer = engine_transfer_completion(engine, transfer); return transfer; } -static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +static int engine_service_perf(struct xdma_engine *engine, u32 desc_completed) { - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } /* performance measurement is running? */ if (engine->xdma_perf) { @@ -844,7 +967,7 @@ static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) if (engine->status & XDMA_STAT_DESC_COMPLETED) { engine->xdma_perf->iterations = desc_completed; dbg_perf("transfer->xdma_perf->iterations=%d\n", - engine->xdma_perf->iterations); + engine->xdma_perf->iterations); } /* a descriptor stopped the engine? */ @@ -854,30 +977,35 @@ static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) * wake any XDMA_PERF_IOCTL_STOP waiting for * the performance run to finish */ - wake_up_interruptible(&engine->xdma_perf_wq); + xlx_wake_up(&engine->xdma_perf_wq); dbg_perf("transfer->xdma_perf stopped\n"); } } + return 0; } -static void engine_transfer_dequeue(struct xdma_engine *engine) +static int engine_transfer_dequeue(struct xdma_engine *engine) { struct xdma_transfer *transfer; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } /* pick first transfer on the queue (was submitted to the engine) */ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); - if (!transfer || transfer != &engine->cyclic_req->xfer) { - pr_info("%s, xfer 0x%p != 0x%p.\n", - engine->name, transfer, &engine->cyclic_req->xfer); - return; + entry); + if (!transfer || transfer != &engine->cyclic_req->tfer[1]) { + pr_err("%s, xfer 0x%p != 0x%p.\n", engine->name, transfer, + &engine->cyclic_req->tfer[1]); + return -EINVAL; } dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", engine->name, transfer, transfer->desc_num); /* remove completed transfer from list */ list_del(engine->transfer_list.next); + return 0; } static int engine_ring_process(struct xdma_engine *engine) @@ -886,28 +1014,35 @@ static int engine_ring_process(struct xdma_engine *engine) int start; int eop_count = 0; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + result = engine->cyclic_result; - BUG_ON(!result); + if (!result) { + pr_err("%s Cyclic transfer resources not available.\n", + engine->name); + return -EINVAL; + } /* where we start receiving in the ring buffer */ start = engine->rx_tail; /* iterate through all newly received RX result descriptors */ - dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", - engine->name, engine->rx_tail, result[engine->rx_tail].status, + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", engine->name, + engine->rx_tail, result[engine->rx_tail].status, result[engine->rx_tail].length); while (result[engine->rx_tail].status && !engine->rx_overrun) { /* EOP bit set in result? */ - if (result[engine->rx_tail].status & RX_STATUS_EOP){ + if (result[engine->rx_tail].status & RX_STATUS_EOP) eop_count++; - } /* increment tail pointer */ engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; - dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", - engine->name, engine->rx_head, engine->rx_tail, + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", engine->name, + engine->rx_head, engine->rx_tail, result[engine->rx_tail].status, result[engine->rx_tail].length); @@ -924,14 +1059,23 @@ static int engine_ring_process(struct xdma_engine *engine) static int engine_service_cyclic_polled(struct xdma_engine *engine) { - int eop_count = 0; + int eop_count; int rc = 0; struct xdma_poll_wb *writeback_data; u32 sched_limit = 0; - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } + eop_count = engine->eop_count; writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; while (eop_count == 0) { @@ -948,19 +1092,37 @@ static int engine_service_cyclic_polled(struct xdma_engine *engine) } eop_count = engine_ring_process(engine); + if (eop_count < 0) { + pr_err("Failed to process engine ring\n"); + return eop_count; + } } if (eop_count == 0) { - engine_status_read(engine, 1, 0); + rc = engine_status_read(engine, 1, 0); + if (rc < 0) { + pr_err("Failed to read engine status\n"); + return rc; + } if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { /* transfers on queue? */ - if (!list_empty(&engine->transfer_list)) - engine_transfer_dequeue(engine); + if (!list_empty(&engine->transfer_list)) { + rc = engine_transfer_dequeue(engine); + if (rc < 0) { + pr_err("Failed to dequeue transfer\n"); + return rc; + } + } - engine_service_shutdown(engine); + rc = engine_service_shutdown(engine); + if (rc < 0) { + pr_err("Failed to shutdown engine\n"); + return rc; + } } } - + eop_count--; + engine->eop_count = eop_count; return rc; } @@ -968,39 +1130,62 @@ static int engine_service_cyclic_interrupt(struct xdma_engine *engine) { int eop_count = 0; struct xdma_transfer *xfer; + int rv; - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } - engine_status_read(engine, 1, 0); + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } + rv = engine_status_read(engine, 1, 0); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + return rv; + } eop_count = engine_ring_process(engine); + if (eop_count < 0) { + pr_err("Failed to process engine ring\n"); + return eop_count; + } /* * wake any reader on EOP, as one or more packets are now in * the RX buffer */ - xfer = &engine->cyclic_req->xfer; - if(enable_credit_mp){ - if (eop_count > 0) { - //engine->eop_found = 1; - } - wake_up_interruptible(&xfer->wq); - }else{ + xfer = &engine->cyclic_req->tfer[0]; + if (enable_credit_mp) { + xlx_wake_up(&xfer->wq); + } else { if (eop_count > 0) { /* awake task on transfer's wait queue */ - dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); + dbg_tfr("wake_up_interruptible() due to %d EOP's\n", + eop_count); engine->eop_found = 1; - wake_up_interruptible(&xfer->wq); + xlx_wake_up(&xfer->wq); } } /* engine was running but is no longer busy? */ if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { /* transfers on queue? */ - if (!list_empty(&engine->transfer_list)) - engine_transfer_dequeue(engine); + if (!list_empty(&engine->transfer_list)) { + rv = engine_transfer_dequeue(engine); + if (rv < 0) { + pr_err("Failed to dequeue transfer\n"); + return rv; + } + } - engine_service_shutdown(engine); + rv = engine_service_shutdown(engine); + if (rv < 0) { + pr_err("Failed to shutdown engine\n"); + return rv; + } } return 0; @@ -1011,10 +1196,18 @@ static int engine_service_cyclic(struct xdma_engine *engine) { int rc = 0; - dbg_tfr("engine_service_cyclic()"); + dbg_tfr("%s()\n", __func__); + + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } if (poll_mode) rc = engine_service_cyclic_polled(engine); @@ -1024,12 +1217,14 @@ static int engine_service_cyclic(struct xdma_engine *engine) return rc; } - -static void engine_service_resume(struct xdma_engine *engine) +static int engine_service_resume(struct xdma_engine *engine) { struct xdma_transfer *transfer_started; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } /* engine stopped? */ if (!engine->running) { @@ -1037,13 +1232,17 @@ static void engine_service_resume(struct xdma_engine *engine) if (!list_empty(&engine->transfer_list)) { /* (re)start engine */ transfer_started = engine_start(engine); - pr_info("re-started %s engine with pending xfer 0x%p\n", + if (!transfer_started) { + pr_err("Failed to start dma engine\n"); + return -EINVAL; + } + dbg_tfr("re-started %s engine with pending xfer 0x%p\n", engine->name, transfer_started); - /* engine was requested to be shutdown? */ + /* engine was requested to be shutdown? */ } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { engine->shutdown |= ENGINE_SHUTDOWN_IDLE; /* awake task on engine's shutdown wait queue */ - wake_up_interruptible(&engine->shutdown_wq); + xlx_wake_up(&engine->shutdown_wq); } else { dbg_tfr("no pending transfers, %s engine stays idle.\n", engine->name); @@ -1056,6 +1255,7 @@ static void engine_service_resume(struct xdma_engine *engine) WARN_ON(1); } } + return 0; } /** @@ -1074,7 +1274,10 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) int rv = 0; struct xdma_poll_wb *wb_data; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } /* If polling detected an error, signal to the caller */ if (err_flag) @@ -1083,7 +1286,11 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) /* Service the engine */ if (!engine->running) { dbg_tfr("Engine was not running!!! Clearing status\n"); - engine_status_read(engine, 1, 0); + rv = engine_status_read(engine, 1, 0); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + return rv; + } return 0; } @@ -1092,17 +1299,26 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) * engine status. For polled mode descriptor completion, this read is * unnecessary and is skipped to reduce latency */ - if ((desc_count == 0) || (err_flag != 0)) - engine_status_read(engine, 1, 0); + if ((desc_count == 0) || (err_flag != 0)) { + rv = engine_status_read(engine, 1, 0); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + return rv; + } + } /* * engine was running but is no longer busy, or writeback occurred, * shut down */ if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || - (desc_count != 0)) - engine_service_shutdown(engine); - + (desc_count != 0)) { + rv = engine_service_shutdown(engine); + if (rv < 0) { + pr_err("Failed to shutdown engine\n"); + return rv; + } + } /* * If called from the ISR, or if an error occurred, the descriptor * count will be zero. In this scenario, read the descriptor count @@ -1117,7 +1333,7 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) if (!list_empty(&engine->transfer_list)) { /* pick first transfer on queue (was submitted to the engine) */ transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); + struct xdma_transfer, entry); dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", transfer, (int)transfer->desc_num); @@ -1126,7 +1342,11 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) (int)desc_count, (int)desc_count - engine->desc_dequeued); - engine_service_perf(engine, desc_count); + rv = engine_service_perf(engine, desc_count); + if (rv < 0) { + pr_err("Failed to service descriptors\n"); + return rv; + } } /* account for already dequeued transfers during this engine run */ @@ -1142,15 +1362,17 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) transfer = engine_service_final_transfer(engine, transfer, &desc_count); /* Before starting engine again, clear the writeback data */ - if (poll_mode) { + if (poll_mode) { wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; wb_data->completed_desc_count = 0; } /* Restart the engine following the servicing */ - engine_service_resume(engine); + rv = engine_service_resume(engine); + if (rv < 0) + pr_err("Failed to resume engine\n"); - return 0; + return rv; } /* engine_service_work */ @@ -1158,50 +1380,68 @@ static void engine_service_work(struct work_struct *work) { struct xdma_engine *engine; unsigned long flags; + int rv; engine = container_of(work, struct xdma_engine, work); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return; + } /* lock the engine */ spin_lock_irqsave(&engine->lock, flags); - dbg_tfr("engine_service() for %s engine %p\n", - engine->name, engine); - if (engine->cyclic_req) - engine_service_cyclic(engine); - else - engine_service(engine, 0); - + dbg_tfr("engine_service() for %s engine %p\n", engine->name, engine); + if (engine->cyclic_req) { + rv = engine_service_cyclic(engine); + if (rv < 0) { + pr_err("Failed to service cyclic engine\n"); + goto unlock; + } + } else { + rv = engine_service(engine, 0); + if (rv < 0) { + pr_err("Failed to service engine\n"); + goto unlock; + } + } /* re-enable interrupts for this engine */ - if (engine->xdev->msix_enabled){ - write_register(engine->interrupt_enable_mask_value, - &engine->regs->interrupt_enable_mask_w1s, - (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - - (unsigned long)(&engine->regs)); + if (engine->xdev->msix_enabled) { + write_register( + engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs + ->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); } else channel_interrupts_enable(engine->xdev, engine->irq_bitmask); - + /* unlock the engine */ +unlock: spin_unlock_irqrestore(&engine->lock, flags); } static u32 engine_service_wb_monitor(struct xdma_engine *engine, - u32 expected_wb) + u32 expected_wb) { struct xdma_poll_wb *wb_data; u32 desc_wb = 0; u32 sched_limit = 0; unsigned long timeout; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; /* - * Poll the writeback location for the expected number of - * descriptors / error events This loop is skipped for cyclic mode, - * where the expected_desc_count passed in is zero, since it cannot be - * determined before the function is called - */ + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); while (expected_wb != 0) { @@ -1209,10 +1449,9 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, if (desc_wb & WB_ERR_MASK) break; - else if (desc_wb == expected_wb) + else if (desc_wb >= expected_wb) break; - - /* RTO - prevent system from hanging in polled mode */ + if (time_after(jiffies, timeout)) { dbg_tfr("Polling timeout occurred"); dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, @@ -1224,9 +1463,9 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, } /* - * Define NUM_POLLS_PER_SCHED to limit how much time is spent - * in the scheduler - */ + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ if (sched_limit != 0) { if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) @@ -1238,16 +1477,24 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, return desc_wb; } -static int engine_service_poll(struct xdma_engine *engine, - u32 expected_desc_count) +int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) { struct xdma_poll_wb *writeback_data; u32 desc_wb = 0; unsigned long flags; int rv = 0; - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; @@ -1257,21 +1504,20 @@ static int engine_service_poll(struct xdma_engine *engine, } /* - * Poll the writeback location for the expected number of - * descriptors / error events This loop is skipped for cyclic mode, - * where the expected_desc_count passed in is zero, since it cannot be - * determined before the function is called - */ + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ desc_wb = engine_service_wb_monitor(engine, expected_desc_count); spin_lock_irqsave(&engine->lock, flags); dbg_tfr("%s service.\n", engine->name); - if (engine->cyclic_req) { + if (engine->cyclic_req) rv = engine_service_cyclic(engine); - } else { + else rv = engine_service(engine, desc_wb); - } spin_unlock_irqrestore(&engine->lock, flags); return rv; @@ -1281,7 +1527,10 @@ static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) { unsigned long flags; - BUG_ON(!user_irq); + if (!user_irq) { + pr_err("Invalid user_irq\n"); + return IRQ_NONE; + } if (user_irq->handler) return user_irq->handler(user_irq->user_idx, user_irq->dev); @@ -1310,17 +1559,20 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) struct interrupt_regs *irq_regs; dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); - BUG_ON(!dev_id); + if (!dev_id) { + pr_err("Invalid dev_id on irq line %d\n", irq); + return -IRQ_NONE; + } xdev = (struct xdma_dev *)dev_id; if (!xdev) { WARN_ON(!xdev); - dbg_irq("xdma_isr(irq=%d) xdev=%p ??\n", irq, xdev); + dbg_irq("%s(irq=%d) xdev=%p ??\n", __func__, irq, xdev); return IRQ_NONE; } irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + XDMA_OFS_INT_CTRL); /* read channel interrupt requests */ ch_irq = read_register(&irq_regs->channel_int_request); @@ -1339,8 +1591,8 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) if (user_irq) { int user = 0; - u32 mask = 1; - int max = xdev->h2c_channel_max; + u32 mask = 1; + int max = xdev->user_max; for (; user < max && user_irq; user++, mask <<= 1) { if (user_irq & mask) { @@ -1360,8 +1612,8 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) struct xdma_engine *engine = &xdev->engine_h2c[channel]; /* engine present and its interrupt fired? */ - if((engine->irq_bitmask & mask) && - (engine->magic == MAGIC_ENGINE)) { + if ((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { mask &= ~engine->irq_bitmask; dbg_tfr("schedule_work, %s.\n", engine->name); schedule_work(&engine->work); @@ -1379,8 +1631,8 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) struct xdma_engine *engine = &xdev->engine_c2h[channel]; /* engine present and its interrupt fired? */ - if((engine->irq_bitmask & mask) && - (engine->magic == MAGIC_ENGINE)) { + if ((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { mask &= ~engine->irq_bitmask; dbg_tfr("schedule_work, %s.\n", engine->name); schedule_work(&engine->work); @@ -1403,10 +1655,13 @@ static irqreturn_t xdma_user_irq(int irq, void *dev_id) dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); - BUG_ON(!dev_id); + if (!dev_id) { + pr_err("Invalid dev_id on irq line %d\n", irq); + return IRQ_NONE; + } user_irq = (struct xdma_user_irq *)dev_id; - return user_irq_service(irq, user_irq); + return user_irq_service(irq, user_irq); } /* @@ -1421,35 +1676,34 @@ static irqreturn_t xdma_channel_irq(int irq, void *dev_id) struct interrupt_regs *irq_regs; dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); - BUG_ON(!dev_id); + if (!dev_id) { + pr_err("Invalid dev_id on irq line %d\n", irq); + return IRQ_NONE; + } engine = (struct xdma_engine *)dev_id; xdev = engine->xdev; if (!xdev) { WARN_ON(!xdev); - dbg_irq("xdma_channel_irq(irq=%d) xdev=%p ??\n", irq, xdev); + dbg_irq("%s(irq=%d) xdev=%p ??\n", __func__, irq, xdev); return IRQ_NONE; } irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + XDMA_OFS_INT_CTRL); /* Disable the interrupt for this engine */ - write_register(engine->interrupt_enable_mask_value, - &engine->regs->interrupt_enable_mask_w1c, - (unsigned long) - (&engine->regs->interrupt_enable_mask_w1c) - + write_register( + engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1c) - (unsigned long)(&engine->regs)); /* Dummy read to flush the above write */ read_register(&irq_regs->channel_int_pending); /* Schedule the bottom half */ schedule_work(&engine->work); - /* - * RTO - need to protect access here if multiple MSI-X are used for - * user interrupts - */ xdev->irq_count++; return IRQ_HANDLED; } @@ -1521,28 +1775,29 @@ static int is_config_bar(struct xdma_dev *xdev, int idx) int flag = 0; u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ struct interrupt_regs *irq_regs = - (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); + (struct interrupt_regs *)(xdev->bar[idx] + XDMA_OFS_INT_CTRL); struct config_regs *cfg_regs = (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); irq_id = read_register(&irq_regs->identifier); cfg_id = read_register(&cfg_regs->identifier); - if (((irq_id & mask)== IRQ_BLOCK_ID) && - ((cfg_id & mask)== CONFIG_BLOCK_ID)) { + if (((irq_id & mask) == IRQ_BLOCK_ID) && + ((cfg_id & mask) == CONFIG_BLOCK_ID)) { dbg_init("BAR %d is the XDMA config BAR\n", idx); flag = 1; } else { dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", - idx, irq_id, cfg_id); + idx, irq_id, cfg_id); flag = 0; } return flag; } -static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, - int config_bar_pos) +#ifndef XDMA_CONFIG_BAR_NUM +static int identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) { /* * The following logic identifies which BARs contain what functionality @@ -1558,11 +1813,18 @@ static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, * correctly with both 32-bit and 64-bit BARs. */ - BUG_ON(!xdev); - BUG_ON(!bar_id_list); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } - dbg_init("xdev 0x%p, bars %d, config at %d.\n", - xdev, num_bars, config_bar_pos); + if (!bar_id_list) { + pr_err("Invalid bar id list.\n"); + return -EINVAL; + } + + dbg_init("xdev 0x%p, bars %d, config at %d.\n", xdev, num_bars, + config_bar_pos); switch (num_bars) { case 1: @@ -1598,12 +1860,12 @@ static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", num_bars); break; - } - pr_info("%d BARs: config %d, user %d, bypass %d.\n", - num_bars, config_bar_pos, xdev->user_bar_idx, - xdev->bypass_bar_idx); + pr_info("%d BARs: config %d, user %d, bypass %d.\n", num_bars, + config_bar_pos, xdev->user_bar_idx, xdev->bypass_bar_idx); + return 0; } +#endif /* map_bars() -- map device regions into kernel virtual address space * @@ -1613,6 +1875,24 @@ static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) { int rv; + +#ifdef XDMA_CONFIG_BAR_NUM + rv = map_single_bar(xdev, dev, XDMA_CONFIG_BAR_NUM); + if (rv <= 0) { + pr_info("%s, map config bar %d failed, %d.\n", + dev_name(&dev->dev), XDMA_CONFIG_BAR_NUM, rv); + return -EINVAL; + } + + if (is_config_bar(xdev, XDMA_CONFIG_BAR_NUM) == 0) { + pr_info("%s, unable to identify config bar %d.\n", + dev_name(&dev->dev), XDMA_CONFIG_BAR_NUM); + return -EINVAL; + } + xdev->config_bar_idx = XDMA_CONFIG_BAR_NUM; + + return 0; +#else int i; int bar_id_list[XDMA_BAR_NUM]; int bar_id_idx = 0; @@ -1632,7 +1912,6 @@ static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) /* Try to identify BAR as XDMA control BAR */ if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { - if (is_config_bar(xdev, i)) { xdev->config_bar_idx = i; config_bar_pos = bar_id_idx; @@ -1652,7 +1931,11 @@ static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) goto fail; } - identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); + rv = identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); + if (rv < 0) { + pr_err("Failed to identify bars\n"); + return rv; + } /* successfully mapped all required BAR regions */ return 0; @@ -1661,20 +1944,21 @@ static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) /* unwind; unmap any BARs that we did map */ unmap_bars(xdev, dev); return rv; +#endif } /* - * MSI-X interrupt: + * MSI-X interrupt: * vectors, followed by vectors */ /* - * RTO - code to detect if MSI/MSI-X capability exists is derived + * code to detect if MSI/MSI-X capability exists is derived * from linux/pci/msi.c - pci_msi_check_device */ #ifndef arch_msi_check_device -int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) +static int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) { return 0; } @@ -1718,17 +2002,24 @@ static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) { int rv = 0; - BUG_ON(!xdev); - BUG_ON(!pdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } + + if (!pdev) { + pr_err("Invalid pdev\n"); + return -EINVAL; + } if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + - xdev->user_max; + xdev->user_max; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE dbg_init("Enabling MSI-X\n"); rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, - PCI_IRQ_MSIX); + PCI_IRQ_MSIX); #else int i; @@ -1766,7 +2057,7 @@ static void pci_check_intr_pend(struct pci_dev *pdev) pci_read_config_word(pdev, PCI_STATUS, &v); if (v & PCI_STATUS_INTERRUPT) { pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", - dev_name(&pdev->dev), v); + dev_name(&pdev->dev), v); pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); } } @@ -1792,9 +2083,9 @@ static void pci_keep_intx_enabled(struct pci_dev *pdev) static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) { /* user */ - struct interrupt_regs *int_regs = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *int_regs = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; u32 max = i + xdev->user_max; int j; @@ -1809,21 +2100,22 @@ static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) else for (k = 0; k < 4 && i < max; i++, k++, shift += 8) val |= (i & 0x1f) << shift; - - write_register(val, &int_regs->user_msi_vector[j], + + write_register( + val, &int_regs->user_msi_vector[j], XDMA_OFS_INT_CTRL + - ((unsigned long)&int_regs->user_msi_vector[j] - - (unsigned long)int_regs)); + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); dbg_init("vector %d, 0x%x.\n", j, val); } } -static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) { - struct interrupt_regs *int_regs = (struct interrupt_regs *) - (xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *int_regs = + (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; u32 i; int j; @@ -1839,11 +2131,12 @@ static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) else for (k = 0; k < 4 && i < max; i++, k++, shift += 8) val |= (i & 0x1f) << shift; - + write_register(val, &int_regs->channel_msi_vector[j], - XDMA_OFS_INT_CTRL + - ((unsigned long)&int_regs->channel_msi_vector[j] - - (unsigned long)int_regs)); + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs + ->channel_msi_vector[j] - + (unsigned long)int_regs)); dbg_init("vector %d, 0x%x.\n", j, val); } } @@ -1859,21 +2152,21 @@ static void irq_msix_channel_teardown(struct xdma_dev *xdev) prog_irq_msix_channel(xdev, 1); - engine = xdev->engine_h2c; + engine = xdev->engine_h2c; for (i = 0; i < xdev->h2c_channel_max; i++, j++, engine++) { if (!engine->msix_irq_line) break; dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, - engine); + engine); free_irq(engine->msix_irq_line, engine); } - engine = xdev->engine_c2h; + engine = xdev->engine_c2h; for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { if (!engine->msix_irq_line) break; dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, - engine); + engine); free_irq(engine->msix_irq_line, engine); } } @@ -1881,18 +2174,23 @@ static void irq_msix_channel_teardown(struct xdma_dev *xdev) static int irq_msix_channel_setup(struct xdma_dev *xdev) { int i; - int j = xdev->h2c_channel_max; + int j; int rv = 0; u32 vector; struct xdma_engine *engine; - BUG_ON(!xdev); + if (!xdev) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + if (!xdev->msix_enabled) return 0; + j = xdev->h2c_channel_max; engine = xdev->engine_h2c; for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE vector = pci_irq_vector(xdev->pdev, i); #else vector = xdev->entry[i].vector; @@ -1910,7 +2208,7 @@ static int irq_msix_channel_setup(struct xdma_dev *xdev) engine = xdev->engine_c2h; for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE vector = pci_irq_vector(xdev->pdev, j); #else vector = xdev->entry[j].vector; @@ -1932,17 +2230,22 @@ static int irq_msix_channel_setup(struct xdma_dev *xdev) static void irq_msix_user_teardown(struct xdma_dev *xdev) { int i; - int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + int j; - BUG_ON(!xdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return; + } if (!xdev->msix_enabled) return; + j = xdev->h2c_channel_max + xdev->c2h_channel_max; + prog_irq_msix_user(xdev, 1); for (i = 0; i < xdev->user_max; i++, j++) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; @@ -1956,30 +2259,30 @@ static int irq_msix_user_setup(struct xdma_dev *xdev) { int i; int j = xdev->h2c_channel_max + xdev->c2h_channel_max; - int rv = 0; + int rv = 0; /* vectors set in probe_scan_for_msi() */ for (i = 0; i < xdev->user_max; i++, j++) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; #endif rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, - &xdev->user_irq[i]); + &xdev->user_irq[i]); if (rv) { - pr_info("user %d couldn't use IRQ#%d, %d\n", - i, vector, rv); + pr_info("user %d couldn't use IRQ#%d, %d\n", i, vector, + rv); break; } pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, &xdev->user_irq[i]); - } + } /* If any errors occur, free IRQs that were successfully requested */ if (rv) { for (i--, j--; i >= 0; i--, j--) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; @@ -2016,22 +2319,23 @@ static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) dbg_init("Legacy Interrupt register value = %d\n", val); if (val > 1) { val--; - w = (val<<24) | (val<<16) | (val<<8)| val; + w = (val << 24) | (val << 16) | (val << 8) | val; /* Program IRQ Block Channel vactor and IRQ Block User vector - * with Legacy interrupt value */ - reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + * with Legacy interrupt value + */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user write_register(w, reg, 0x2080); - write_register(w, reg+0x4, 0x2084); - write_register(w, reg+0x8, 0x2088); - write_register(w, reg+0xC, 0x208C); - reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + write_register(w, reg + 0x4, 0x2084); + write_register(w, reg + 0x8, 0x2088); + write_register(w, reg + 0xC, 0x208C); + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block write_register(w, reg, 0x20A0); - write_register(w, reg+0x4, 0x20A4); + write_register(w, reg + 0x4, 0x20A4); } xdev->irq_line = (int)pdev->irq; rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, - xdev); + xdev); if (rv) dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); else @@ -2057,6 +2361,7 @@ static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) if (xdev->msix_enabled) { int rv = irq_msix_channel_setup(xdev); + if (rv) return rv; rv = irq_msix_user_setup(xdev); @@ -2077,19 +2382,23 @@ static void dump_desc(struct xdma_desc *desc_virt) { int j; u32 *p = (u32 *)desc_virt; - static char * const field_name[] = { - "magic|extra_adjacent|control", "bytes", "src_addr_lo", - "src_addr_hi", "dst_addr_lo", "dst_addr_hi", "next_addr", - "next_addr_pad"}; + static char *const field_name[] = { "magic|extra_adjacent|control", + "bytes", + "src_addr_lo", + "src_addr_hi", + "dst_addr_lo", + "dst_addr_hi", + "next_addr", + "next_addr_pad" }; char *dummy; /* remove warning about unused variable when debug printing is off */ dummy = field_name[0]; for (j = 0; j < 8; j += 1) { - pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", - (uintptr_t)p, (uintptr_t)p & 15, (int)*p, - le32_to_cpu(*p), field_name[j]); + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", (uintptr_t)p, + (uintptr_t)p & 15, (int)*p, le32_to_cpu(*p), + field_name[j]); p++; } pr_info("\n"); @@ -2104,38 +2413,37 @@ static void transfer_dump(struct xdma_transfer *transfer) transfer, transfer->state, transfer->flags, transfer->dir, transfer->len, transfer->last_in_request); - pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", - transfer, transfer->desc_num, (u64)transfer->desc_bus, + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", transfer, + transfer->desc_num, (u64)transfer->desc_bus, transfer->desc_adjacent); for (i = 0; i < transfer->desc_num; i += 1) dump_desc(desc_virt + i); } #endif /* __LIBXDMA_DEBUG__ */ -/* xdma_desc_alloc() - Allocate cache-coherent array of N descriptors. - * - * Allocates an array of 'number' descriptors in contiguous PCI bus addressable - * memory. Chains the descriptors as a singly-linked list; the descriptor's - * next * pointer specifies the bus address of the next descriptor. +/* transfer_desc_init() - Chains the descriptors as a singly-linked list * + * Each descriptor's next * pointer specifies the bus address + * of the next descriptor. + * Terminates the last descriptor to form a singly-linked list * - * @dev Pointer to pci_dev - * @number Number of descriptors to be allocated - * @desc_bus_p Pointer where to store the first descriptor bus address - * - * @return Virtual address of the first descriptor + * @transfer Pointer to SG DMA transfers + * @count Number of descriptors allocated in continuous PCI bus addressable + * memory * + * @return 0 on success, EINVAL on failure */ -static void transfer_desc_init(struct xdma_transfer *transfer, int count) +static int transfer_desc_init(struct xdma_transfer *transfer, int count) { struct xdma_desc *desc_virt = transfer->desc_virt; dma_addr_t desc_bus = transfer->desc_bus; int i; - int adj = count - 1; - int extra_adj; - u32 temp_control; - BUG_ON(count > XDMA_TRANSFER_MAX_DESC); + if (count > XDMA_TRANSFER_MAX_DESC) { + pr_err("Engine cannot transfer more than %d descriptors\n", + XDMA_TRANSFER_MAX_DESC); + return -EINVAL; + } /* create singly-linked list for SG DMA controller */ for (i = 0; i < count - 1; i++) { @@ -2146,31 +2454,16 @@ static void transfer_desc_init(struct xdma_transfer *transfer, int count) desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); desc_virt[i].bytes = cpu_to_le32(0); - - /* any adjacent descriptors? */ - if (adj > 0) { - extra_adj = adj - 1; - if (extra_adj > MAX_EXTRA_ADJ) - extra_adj = MAX_EXTRA_ADJ; - - adj--; - } else { - extra_adj = 0; - } - - temp_control = DESC_MAGIC | (extra_adj << 8); - - desc_virt[i].control = cpu_to_le32(temp_control); + desc_virt[i].control = cpu_to_le32(DESC_MAGIC); } /* { i = number - 1 } */ /* zero the last descriptor next pointer */ desc_virt[i].next_lo = cpu_to_le32(0); desc_virt[i].next_hi = cpu_to_le32(0); desc_virt[i].bytes = cpu_to_le32(0); + desc_virt[i].control = cpu_to_le32(DESC_MAGIC); - temp_control = DESC_MAGIC; - - desc_virt[i].control = cpu_to_le32(temp_control); + return 0; } /* xdma_desc_link() - Link two descriptors @@ -2182,13 +2475,12 @@ static void transfer_desc_init(struct xdma_transfer *transfer, int count) * @second_bus bus address of second descriptor */ static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, - dma_addr_t second_bus) + dma_addr_t second_bus) { /* * remember reserved control in first descriptor, but zero * extra_adjacent! */ - /* RTO - what's this about? Shouldn't it be 0x0000c0ffUL? */ u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; /* second descriptor given? */ if (second) { @@ -2215,56 +2507,51 @@ static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, /* xdma_desc_adjacent -- Set how many descriptors are adjacent to this one */ static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) { - int extra_adj = 0; /* remember reserved and control bits */ - u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL; - u32 max_adj_4k = 0; + u32 control = le32_to_cpu(desc->control) & 0xffffc0ffUL; - if (next_adjacent > 0) { - extra_adj = next_adjacent - 1; - if (extra_adj > MAX_EXTRA_ADJ){ - extra_adj = MAX_EXTRA_ADJ; - } - max_adj_4k = (0x1000 - ((le32_to_cpu(desc->next_lo))&0xFFF))/32 - 1; - if (extra_adj>max_adj_4k) { - extra_adj = max_adj_4k; - } - if(extra_adj<0){ - printk("Warning: extra_adj<0, converting it to 0\n"); - extra_adj = 0; - } - } - /* merge adjacent and control field */ - control |= 0xAD4B0000UL | (extra_adj << 8); + if (next_adjacent) + next_adjacent = next_adjacent - 1; + if (next_adjacent > MAX_EXTRA_ADJ) + next_adjacent = MAX_EXTRA_ADJ; + control |= (next_adjacent << 8); /* write control and next_adjacent */ desc->control = cpu_to_le32(control); } /* xdma_desc_control -- Set complete control field of a descriptor. */ -static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field) +static int xdma_desc_control_set(struct xdma_desc *first, u32 control_field) { /* remember magic and adjacent number */ u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); - BUG_ON(control_field & ~(LS_BYTE_MASK)); + if (control_field & ~(LS_BYTE_MASK)) { + pr_err("Invalid control field\n"); + return -EINVAL; + } /* merge adjacent and control field */ control |= control_field; /* write control and next_adjacent */ first->control = cpu_to_le32(control); + return 0; } /* xdma_desc_clear -- Clear bits in control field of a descriptor. */ -static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +static int xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) { /* remember magic and adjacent number */ u32 control = le32_to_cpu(first->control); - BUG_ON(clear_mask & ~(LS_BYTE_MASK)); + if (clear_mask & ~(LS_BYTE_MASK)) { + pr_err("Invalid clear mask\n"); + return -EINVAL; + } /* merge adjacent and control field */ control &= (~clear_mask); /* write control and next_adjacent */ first->control = cpu_to_le32(control); + return 0; } /* xdma_desc_done - recycle cache-coherent linked list of descriptors. @@ -2274,9 +2561,9 @@ static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list * @desc_bus Bus address of first descriptor in list */ -static inline void xdma_desc_done(struct xdma_desc *desc_virt) +static inline void xdma_desc_done(struct xdma_desc *desc_virt, int count) { - memset(desc_virt, 0, XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc)); + memset(desc_virt, 0, count * sizeof(struct xdma_desc)); } /* xdma_desc() - Fill a descriptor with the transfer details @@ -2291,7 +2578,7 @@ static inline void xdma_desc_done(struct xdma_desc *desc_virt) * Does not modify the next pointer */ static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, - u64 ep_addr, int len, int dir) + u64 ep_addr, int len, int dir) { /* transfer length */ desc->bytes = cpu_to_le32(len); @@ -2315,28 +2602,41 @@ static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, /* * should hold the engine->lock; */ -static void transfer_abort(struct xdma_engine *engine, - struct xdma_transfer *transfer) +static int transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) { struct xdma_transfer *head; - BUG_ON(!engine); - BUG_ON(!transfer); - BUG_ON(transfer->desc_num == 0); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (!transfer) { + pr_err("Invalid DMA transfer\n"); + return -EINVAL; + } + + if (transfer->desc_num == 0) { + pr_err("%s void descriptors in the transfer list\n", + engine->name); + return -EINVAL; + } pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", transfer, transfer->desc_num, engine->desc_dequeued); head = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); + entry); if (head == transfer) list_del(engine->transfer_list.next); - else + else pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", engine->name, transfer, head); if (transfer->state == TRANSFER_STATE_SUBMITTED) transfer->state = TRANSFER_STATE_ABORTED; + return 0; } /* transfer_queue() - Queue a DMA transfer on the engine @@ -2347,23 +2647,39 @@ static void transfer_abort(struct xdma_engine *engine, * Takes and releases the engine spinlock */ static int transfer_queue(struct xdma_engine *engine, - struct xdma_transfer *transfer) + struct xdma_transfer *transfer) { int rv = 0; struct xdma_transfer *transfer_started; struct xdma_dev *xdev; unsigned long flags; - BUG_ON(!engine); - BUG_ON(!engine->xdev); - BUG_ON(!transfer); - BUG_ON(transfer->desc_num == 0); - dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (!engine->xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } + + if (!transfer) { + pr_err("%s Invalid DMA transfer\n", engine->name); + return -EINVAL; + } + + if (transfer->desc_num == 0) { + pr_err("%s void descriptors in the transfer list\n", + engine->name); + return -EINVAL; + } + dbg_tfr("%s (transfer=0x%p).\n", __func__, transfer); xdev = engine->xdev; if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { - pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", - xdev, transfer); + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", xdev, + transfer); return -EBUSY; } @@ -2389,9 +2705,12 @@ static int transfer_queue(struct xdma_engine *engine, /* engine is idle? */ if (!engine->running) { /* start engine */ - dbg_tfr("transfer_queue(): starting %s engine.\n", - engine->name); + dbg_tfr("%s(): starting %s engine.\n", __func__, engine->name); transfer_started = engine_start(engine); + if (!transfer_started) { + pr_err("Failed to start dma engine\n"); + goto shutdown; + } dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", transfer, engine->name, transfer_started); } else { @@ -2414,10 +2733,9 @@ static void engine_alignments(struct xdma_engine *engine) u32 address_bits; w = read_register(&engine->regs->alignments); - dbg_init("engine %p name %s alignments=0x%08x\n", engine, - engine->name, (int)w); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, engine->name, + (int)w); - /* RTO - add some macros to extract these fields */ align_bytes = (w & 0x00ff0000U) >> 16; granularity_bytes = (w & 0x0000ff00U) >> 8; address_bits = (w & 0x000000ffU); @@ -2445,59 +2763,72 @@ static void engine_free_resource(struct xdma_engine *engine) /* Release memory use for descriptor writebacks */ if (engine->poll_mode_addr_virt) { dbg_sg("Releasing memory for descriptor writeback\n"); - dma_free_coherent(&xdev->pdev->dev, - sizeof(struct xdma_poll_wb), - engine->poll_mode_addr_virt, - engine->poll_mode_bus); + dma_free_coherent(&xdev->pdev->dev, sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); dbg_sg("Released memory for descriptor writeback\n"); engine->poll_mode_addr_virt = NULL; } if (engine->desc) { dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", - dev_name(&xdev->pdev->dev), engine->name, - engine->desc, engine->desc_bus); + dev_name(&xdev->pdev->dev), engine->name, engine->desc, + engine->desc_bus); dma_free_coherent(&xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), - engine->desc, engine->desc_bus); + XDMA_TRANSFER_MAX_DESC * + sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); engine->desc = NULL; } if (engine->cyclic_result) { - dma_free_coherent(&xdev->pdev->dev, - CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + dma_free_coherent( + &xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result), engine->cyclic_result, engine->cyclic_result_bus); engine->cyclic_result = NULL; } } -static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +static int engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) { - BUG_ON(!xdev); - BUG_ON(!engine); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } + + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); /* Disable interrupts to stop processing new events during shutdown */ write_register(0x0, &engine->regs->interrupt_enable_mask, - (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); if (enable_credit_mp && engine->streaming && - engine->dir == DMA_FROM_DEVICE) { + engine->dir == DMA_FROM_DEVICE) { u32 reg_value = (0x1 << engine->channel) << 16; - struct sgdma_common_regs *reg = (struct sgdma_common_regs *) - (xdev->bar[xdev->config_bar_idx] + - (0x6*TARGET_SPACING)); + struct sgdma_common_regs *reg = + (struct sgdma_common_regs + *)(xdev->bar[xdev->config_bar_idx] + + (0x6 * TARGET_SPACING)); write_register(reg_value, ®->credit_mode_enable_w1c, 0); } + if (poll_mode) + xdma_thread_remove_work(engine); + /* Release memory use for descriptor writebacks */ engine_free_resource(engine); memset(engine, 0, sizeof(struct xdma_engine)); /* Decrement the number of engines available */ xdev->engines_num--; + return 0; } /** @@ -2507,36 +2838,44 @@ static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) */ struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) { + int rv; struct xdma_transfer *transfer = 0; + int size = engine->xdma_perf->transfer_size; /* transfers on queue? */ if (!list_empty(&engine->transfer_list)) { /* pick first transfer on the queue (was submitted to engine) */ transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); - BUG_ON(!transfer); - - xdma_engine_stop(engine); + struct xdma_transfer, entry); + if (!transfer) { + pr_err("(engine=%s) has void transfer in queue.\n", + engine->name); + return NULL; + } + rv = xdma_engine_stop(engine); + if (rv < 0) { + pr_err("Failed to stop engine\n"); + return NULL; + } + engine->running = 0; if (transfer->cyclic) { if (engine->xdma_perf) dbg_perf("Stopping perf transfer on %s\n", - engine->name); + engine->name); else dbg_perf("Stopping cyclic transfer on %s\n", - engine->name); - /* make sure the handler sees correct transfer state */ - transfer->cyclic = 1; - /* - * set STOP flag and interrupt on completion, on the - * last descriptor - */ - xdma_desc_control_set( - transfer->desc_virt + transfer->desc_num - 1, - XDMA_DESC_COMPLETED | XDMA_DESC_STOPPED); + engine->name); + /* free up the buffer allocated for perf run */ + if (engine->perf_buf_virt) + dma_free_coherent(&engine->xdev->pdev->dev, + size, engine->perf_buf_virt, + engine->perf_buf_bus); + engine->perf_buf_virt = NULL; + list_del(&transfer->entry); } else { dbg_sg("(engine=%p) running transfer is not cyclic\n", - engine); + engine); } } else { dbg_sg("(engine=%p) found not running transfer.\n", engine); @@ -2551,33 +2890,34 @@ static int engine_writeback_setup(struct xdma_engine *engine) struct xdma_dev *xdev; struct xdma_poll_wb *writeback; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + xdev = engine->xdev; - BUG_ON(!xdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } - /* - * RTO - doing the allocation per engine is wasteful since a full page - * is allocated each time - better to allocate one page for the whole - * device during probe() and set per-engine offsets here - */ writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; writeback->completed_desc_count = 0; dbg_init("Setting writeback location to 0x%llx for engine %p", - engine->poll_mode_bus, engine); + engine->poll_mode_bus, engine); w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); - write_register(w, &engine->regs->poll_mode_wb_lo, - (unsigned long)(&engine->regs->poll_mode_wb_lo) - - (unsigned long)(&engine->regs)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); - write_register(w, &engine->regs->poll_mode_wb_hi, - (unsigned long)(&engine->regs->poll_mode_wb_hi) - - (unsigned long)(&engine->regs)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); return 0; } - /* engine_create() - Create an SG DMA engine bookkeeping data structure * * An SG DMA engine consists of the resources for a single-direction transfer @@ -2595,8 +2935,8 @@ static int engine_init_regs(struct xdma_engine *engine) int rv = 0; write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, - (unsigned long)(&engine->regs->control_w1c) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); engine_alignments(engine); @@ -2612,34 +2952,31 @@ static int engine_init_regs(struct xdma_engine *engine) rv = engine_writeback_setup(engine); if (rv) { dbg_init("%s descr writeback setup failed.\n", - engine->name); + engine->name); goto fail_wb; } } else { /* enable the relevant completion interrupts */ reg_value |= XDMA_CTRL_IE_DESC_STOPPED; reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; - - if (engine->streaming && engine->dir == DMA_FROM_DEVICE) - reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; } /* Apply engine configurations */ write_register(reg_value, &engine->regs->interrupt_enable_mask, - (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); engine->interrupt_enable_mask_value = reg_value; /* only enable credit mode for AXI-ST C2H */ if (enable_credit_mp && engine->streaming && - engine->dir == DMA_FROM_DEVICE) { - + engine->dir == DMA_FROM_DEVICE) { struct xdma_dev *xdev = engine->xdev; u32 reg_value = (0x1 << engine->channel) << 16; - struct sgdma_common_regs *reg = (struct sgdma_common_regs *) - (xdev->bar[xdev->config_bar_idx] + - (0x6*TARGET_SPACING)); + struct sgdma_common_regs *reg = + (struct sgdma_common_regs + *)(xdev->bar[xdev->config_bar_idx] + + (0x6 * TARGET_SPACING)); write_register(reg_value, ®->credit_mode_enable_w1s, 0); } @@ -2655,8 +2992,9 @@ static int engine_alloc_resource(struct xdma_engine *engine) struct xdma_dev *xdev = engine->xdev; engine->desc = dma_alloc_coherent(&xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), - &engine->desc_bus, GFP_KERNEL); + XDMA_TRANSFER_MAX_DESC * + sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); if (!engine->desc) { pr_warn("dev %s, %s pre-alloc desc OOM.\n", dev_name(&xdev->pdev->dev), engine->name); @@ -2664,24 +3002,25 @@ static int engine_alloc_resource(struct xdma_engine *engine) } if (poll_mode) { - engine->poll_mode_addr_virt = dma_alloc_coherent( - &xdev->pdev->dev, - sizeof(struct xdma_poll_wb), - &engine->poll_mode_bus, GFP_KERNEL); + engine->poll_mode_addr_virt = + dma_alloc_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); if (!engine->poll_mode_addr_virt) { - pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", dev_name(&xdev->pdev->dev), engine->name); goto err_out; } } if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { - engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, - CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + engine->cyclic_result = dma_alloc_coherent( + &xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result), &engine->cyclic_result_bus, GFP_KERNEL); if (!engine->cyclic_result) { - pr_warn("%s, %s pre-alloc result OOM.\n", + pr_warn("%s, %s pre-alloc result OOM.\n", dev_name(&xdev->pdev->dev), engine->name); goto err_out; } @@ -2695,7 +3034,7 @@ static int engine_alloc_resource(struct xdma_engine *engine) } static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, - int offset, enum dma_data_direction dir, int channel) + int offset, enum dma_data_direction dir, int channel) { int rv; u32 val; @@ -2717,9 +3056,9 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, /* register address */ engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + - SGDMA_OFFSET_FROM_CHANNEL; + SGDMA_OFFSET_FROM_CHANNEL; val = read_register(&engine->regs->identifier); - if (val & 0x8000U) + if (val & 0x8000U) engine->streaming = 1; /* remember SG DMA direction */ @@ -2729,7 +3068,7 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, engine->streaming ? "ST" : "MM"); dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, - (int)engine->irq_bitmask); + (int)engine->irq_bitmask); /* initialize the deferred work for transfer completion */ INIT_WORK(&engine->work, engine_service_work); @@ -2748,89 +3087,196 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, if (rv) return rv; + if (poll_mode) + xdma_thread_add_work(engine); + return 0; } /* transfer_destroy() - free transfer */ static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) { - /* free descriptors */ - xdma_desc_done(xfer->desc_virt); + /* free descriptors */ + xdma_desc_done(xfer->desc_virt, xfer->desc_num); if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { - struct sg_table *sgt = xfer->sgt; + struct sg_table *sgt = xfer->sgt; if (sgt->nents) { pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, - xfer->dir); + xfer->dir); sgt->nents = 0; } } } static int transfer_build(struct xdma_engine *engine, - struct xdma_request_cb *req, unsigned int desc_max) + struct xdma_request_cb *req, struct xdma_transfer *xfer, + unsigned int desc_max) { - struct xdma_transfer *xfer = &req->xfer; struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); int i = 0; int j = 0; + dma_addr_t bus = xfer->res_bus; for (; i < desc_max; i++, j++, sdesc++) { dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", - i + req->sw_desc_idx, req->sw_desc_cnt, - sdesc->addr, sdesc->len, req->ep_addr); + i + req->sw_desc_idx, req->sw_desc_cnt, sdesc->addr, + sdesc->len, req->ep_addr); /* fill in descriptor entry j with transfer details */ xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, - sdesc->len, xfer->dir); + sdesc->len, xfer->dir); xfer->len += sdesc->len; /* for non-inc-add mode don't increment ep_addr */ if (!engine->non_incr_addr) req->ep_addr += sdesc->len; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + memset(xfer->res_virt + j, 0, + sizeof(struct xdma_result)); + xfer->desc_virt[j].src_addr_lo = + cpu_to_le32(PCI_DMA_L(bus)); + xfer->desc_virt[j].src_addr_hi = + cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } + } - req->sw_desc_idx += desc_max; + req->sw_desc_idx += desc_max; return 0; } -static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) + +static int transfer_init(struct xdma_engine *engine, + struct xdma_request_cb *req, struct xdma_transfer *xfer) { - struct xdma_transfer *xfer = &req->xfer; unsigned int desc_max = min_t(unsigned int, req->sw_desc_cnt - req->sw_desc_idx, XDMA_TRANSFER_MAX_DESC); + unsigned int desc_align = 0; int i = 0; int last = 0; u32 control; + unsigned long flags; memset(xfer, 0, sizeof(*xfer)); + /* lock the engine state */ + spin_lock_irqsave(&engine->lock, flags); /* initialize wait queue */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) + init_swait_queue_head(&xfer->wq); +#else init_waitqueue_head(&xfer->wq); +#endif /* remember direction of transfer */ xfer->dir = engine->dir; + xfer->desc_virt = engine->desc + engine->desc_idx; + xfer->res_virt = engine->cyclic_result + engine->desc_idx; + xfer->desc_bus = engine->desc_bus + + (sizeof(struct xdma_desc) * engine->desc_idx); + xfer->res_bus = engine->cyclic_result_bus + + (sizeof(struct xdma_result) * engine->desc_idx); + xfer->desc_index = engine->desc_idx; - xfer->desc_virt = engine->desc; - xfer->desc_bus = engine->desc_bus; + /* TODO: Need to handle desc_used >= XDMA_TRANSFER_MAX_DESC */ + + if ((engine->desc_idx + desc_max) >= XDMA_TRANSFER_MAX_DESC) + desc_max = XDMA_TRANSFER_MAX_DESC - engine->desc_idx; transfer_desc_init(xfer, desc_max); - - dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); - transfer_build(engine, req, desc_max); + dbg_sg("xfer= %p transfer->desc_bus = 0x%llx.\n", + xfer, (u64)xfer->desc_bus); + transfer_build(engine, req, xfer, desc_max); + + /* + * Contiguous descriptors cannot cross PAGE boundary + * The 1st descriptor may start in the middle of the page, + * calculate the 1st block of adj desc accordingly + */ + desc_align = 128 - (engine->desc_idx % 128) - 1; + if (desc_align > (desc_max - 1)) + desc_align = desc_max - 1; + + xfer->desc_adjacent = desc_align; /* terminate last descriptor */ last = desc_max - 1; - xdma_desc_link(xfer->desc_virt + last, 0, 0); /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ control = XDMA_DESC_STOPPED; control |= XDMA_DESC_EOP; control |= XDMA_DESC_COMPLETED; xdma_desc_control_set(xfer->desc_virt + last, control); - xfer->desc_num = xfer->desc_adjacent = desc_max; + xfer->desc_num = desc_max; + engine->desc_idx = (engine->desc_idx + desc_max) % + XDMA_TRANSFER_MAX_DESC; + engine->desc_used += desc_max; + + /* fill in adjacent numbers */ + for (i = 0; i < xfer->desc_num && desc_align; i++, desc_align--) + xdma_desc_adjacent(xfer->desc_virt + i, desc_align); + + for (; i < xfer->desc_num; i++) + xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); + + spin_unlock_irqrestore(&engine->lock, flags); + return 0; +} + + +static int transfer_init_cyclic(struct xdma_engine *engine, + struct xdma_request_cb *req, + struct xdma_transfer *xfer) +{ + unsigned int desc_max = + min_t(unsigned int, req->sw_desc_cnt - req->sw_desc_idx, + XDMA_TRANSFER_MAX_DESC); + int i = 0; + u32 control; + int rv; + + memset(xfer, 0, sizeof(*xfer)); + + /* initialize wait queue */ +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + init_swait_queue_head(&xfer->wq); +#else + init_waitqueue_head(&xfer->wq); +#endif + + /* remember direction of transfer */ + xfer->dir = engine->dir; + + xfer->desc_virt = engine->desc; + xfer->desc_bus = engine->desc_bus; + + rv = transfer_desc_init(xfer, desc_max); + if (rv < 0) { + pr_err("Failed to initialize descriptors\n"); + return rv; + } + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); + + transfer_build(engine, req, xfer, desc_max); + + /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ + control = XDMA_DESC_STOPPED; + control |= XDMA_DESC_EOP; + control |= XDMA_DESC_COMPLETED; + rv = xdma_desc_control_set(xfer->desc_virt + desc_max - 1, control); + if (rv < 0) { + pr_err("Failed to set desc control\n"); + return rv; + } + + xfer->desc_num = desc_max; + xfer->desc_adjacent = 1; dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); /* fill in adjacent numbers */ @@ -2846,13 +3292,13 @@ static void sgt_dump(struct sg_table *sgt) int i; struct scatterlist *sg = sgt->sgl; - pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", - sgt, sgt->sgl, sgt->nents, sgt->orig_nents); + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", sgt, sgt->sgl, sgt->nents, + sgt->orig_nents); for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) - pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", - i, sg, sg_page(sg), sg->offset, sg->length, - sg_dma_address(sg), sg_dma_len(sg)); + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", i, sg, + sg_page(sg), sg->offset, sg->length, sg_dma_address(sg), + sg_dma_len(sg)); } static void xdma_request_cb_dump(struct xdma_request_cb *req) @@ -2863,9 +3309,8 @@ static void xdma_request_cb_dump(struct xdma_request_cb *req) req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); sgt_dump(req->sgt); for (i = 0; i < req->sw_desc_cnt; i++) - pr_info("%d/%u, 0x%llx, %u.\n", - i, req->sw_desc_cnt, req->sdesc[i].addr, - req->sdesc[i].len); + pr_info("%d/%u, 0x%llx, %u.\n", i, req->sw_desc_cnt, + req->sdesc[i].addr, req->sdesc[i].len); } #endif @@ -2878,11 +3323,11 @@ static void xdma_request_free(struct xdma_request_cb *req) kfree(req); } -static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) +static struct xdma_request_cb *xdma_request_alloc(unsigned int sdesc_nr) { struct xdma_request_cb *req; unsigned int size = sizeof(struct xdma_request_cb) + - sdesc_nr * sizeof(struct sw_desc); + sdesc_nr * sizeof(struct sw_desc); req = kzalloc(size, GFP_KERNEL); if (!req) { @@ -2898,8 +3343,8 @@ static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) return req; } -static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, - u64 ep_addr) +static struct xdma_request_cb *xdma_init_request(struct sg_table *sgt, + u64 ep_addr) { struct xdma_request_cb *req; struct scatterlist *sg = sgt->sgl; @@ -2907,26 +3352,26 @@ static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, int extra = 0; int i, j = 0; - for (i = 0; i < max; i++, sg = sg_next(sg)) { + for (i = 0; i < max; i++, sg = sg_next(sg)) { unsigned int len = sg_dma_len(sg); if (unlikely(len > desc_blen_max)) extra += (len + desc_blen_max - 1) / desc_blen_max; } -//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); + dbg_tfr("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); max += extra; req = xdma_request_alloc(max); if (!req) return NULL; - req->sgt = sgt; + req->sgt = sgt; req->ep_addr = ep_addr; - for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { unsigned int tlen = sg_dma_len(sg); - dma_addr_t addr = sg_dma_address(sg); + dma_addr_t addr = sg_dma_address(sg); req->total_len += tlen; while (tlen) { @@ -2934,7 +3379,7 @@ static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, if (tlen > desc_blen_max) { req->sdesc[j].len = desc_blen_max; addr += desc_blen_max; - tlen -= desc_blen_max; + tlen -= desc_blen_max; } else { req->sdesc[j].len = tlen; tlen = 0; @@ -2942,22 +3387,392 @@ static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, j++; } } - BUG_ON(j > max); - req->sw_desc_cnt = j; -#ifdef __LIBXDMA_DEBUG__ - xdma_request_cb_dump(req); -#endif - return req; + if (j > max) { + pr_err("Cannot transfer more than supported length %d\n", + desc_blen_max); + xdma_request_free(req); + return NULL; + } + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; +} + +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int rv = 0, tfer_idx = 0, i; + ssize_t done = 0; + struct scatterlist *sg = sgt->sgl; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + struct xdma_result *result; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_err("H2C channel %d >= %d.\n", channel, + xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_err("C2H channel %d >= %d.\n", channel, + xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } + + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("xdev 0x%p, offline.\n", xdev); + return -EBUSY; + } + + /* check the direction */ + if (engine->dir != dir) { + pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", engine, + engine->name, channel, write, engine->dir, dir); + return -EINVAL; + } + + if (!dma_mapped) { + nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); + if (!nents) { + pr_info("map sgl failed, sgt 0x%p.\n", sgt); + return -EIO; + } + sgt->nents = nents; + } else { + if (!sgt->nents) { + pr_err("sg table has invalid number of entries 0x%p.\n", + sgt); + return -EIO; + } + } + + req = xdma_init_request(sgt, ep_addr); + if (!req) { + rv = -ENOMEM; + goto unmap_sgl; + } + + dbg_tfr("%s, len %u sg cnt %u.\n", engine->name, req->total_len, + req->sw_desc_cnt); + + sg = sgt->sgl; + nents = req->sw_desc_cnt; + mutex_lock(&engine->desc_lock); + + while (nents) { + unsigned long flags; + struct xdma_transfer *xfer; + + /* build transfer */ + rv = transfer_init(engine, req, &req->tfer[0]); + if (rv < 0) { + mutex_unlock(&engine->desc_lock); + goto unmap_sgl; + } + xfer = &req->tfer[0]; + + if (!dma_mapped) + xfer->flags = XFER_FLAG_NEED_UNMAP; + + /* last transfer for the given request? */ + nents -= xfer->desc_num; + if (!nents) { + xfer->last_in_request = 1; + xfer->sgt = sgt; + } + + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", xfer->len, + req->ep_addr, done, req->sw_desc_idx, req->sw_desc_cnt); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + rv = transfer_queue(engine, xfer); + if (rv < 0) { + mutex_unlock(&engine->desc_lock); + pr_info("unable to submit %s, %d.\n", engine->name, rv); + goto unmap_sgl; + } + + /* + * When polling, determine how many descriptors have been queued + * on the engine to determine the writeback value expected + */ + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + dbg_tfr("%s poll desc_count=%d\n", engine->name, + desc_count); + rv = engine_service_poll(engine, desc_count); + if (rv < 0) { + mutex_unlock(&engine->desc_lock); + pr_err("Failed to service polling\n"); + goto unmap_sgl; + } + + } else { + xlx_wait_event_interruptible_timeout( + xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch (xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + result = xfer->res_virt; + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, + done); + + /* For C2H streaming use writeback results */ + if (engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + for (i = 0; i < xfer->desc_num; i++) + done += result[i].length; + } else + done += xfer->len; + + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", xfer, + xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + rv = engine_status_read(engine, 0, 1); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + } else if (rv == 0) { + //engine_status_dump(engine); + rv = transfer_abort(engine, xfer); + if (rv < 0) { + pr_err("Failed to stop engine\n"); + } else if (rv == 0) { + rv = xdma_engine_stop(engine); + if (rv < 0) + pr_err("Failed to stop engine\n"); + } + } + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + engine->desc_used -= xfer->desc_num; + transfer_destroy(xdev, xfer); + + /* use multiple transfers per request if we could not fit + * all data within single descriptor chain. + */ + tfer_idx++; + + if (rv < 0) { + mutex_unlock(&engine->desc_lock); + goto unmap_sgl; + } + } /* while (sg) */ + mutex_unlock(&engine->desc_lock); + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + if (rv < 0) + return rv; + + return done; +} +EXPORT_SYMBOL_GPL(xdma_xfer_submit); + +ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, + bool write, u64 ep_addr, struct sg_table *sgt, + bool dma_mapped, int timeout_ms) +{ + + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; + struct xdma_engine *engine; + int rv = 0, tfer_idx = 0; + ssize_t done = 0; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + struct xdma_transfer *xfer; + int i; + struct xdma_result *result; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_warn("H2C channel %d >= %d.\n", + channel, xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_warn("C2H channel %d >= %d.\n", + channel, xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } else { + pr_warn("write %d, exp. 0|1.\n", write); + return -EINVAL; + } + + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } + + xdev = engine->xdev; + req = cb->req; + + nents = req->sw_desc_cnt; + while (nents) { + xfer = &req->tfer[tfer_idx]; + nents -= xfer->desc_num; + switch (xfer->state) { + case TRANSFER_STATE_COMPLETED: + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, + done); + + result = xfer->res_virt; + /* For C2H streaming use writeback results */ + if (engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + for (i = 0; i < xfer->desc_num; i++) + done += result[i].length; + } else + done += xfer->len; + + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + engine_status_dump(engine); + transfer_abort(engine, xfer); + + xdma_engine_stop(engine); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + engine->desc_used -= xfer->desc_num; + + tfer_idx++; + + if (rv < 0) + goto unmap_sgl; + } /* while (sg) */ + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + return done; + } +EXPORT_SYMBOL_GPL(xdma_xfer_completion); -ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, - struct sg_table *sgt, bool dma_mapped, int timeout_ms) + +ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, + bool write, u64 ep_addr, struct sg_table *sgt, + bool dma_mapped, int timeout_ms) { struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; struct xdma_engine *engine; - int rv = 0; - ssize_t done = 0; + struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; + int rv = 0, tfer_idx = 0; struct scatterlist *sg = sgt->sgl; int nents; enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; @@ -2988,8 +3803,16 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, return -EINVAL; } - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", + engine->name, engine->magic); + return -EINVAL; + } xdev = engine->xdev; if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { @@ -3012,7 +3835,11 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, } sgt->nents = nents; } else { - BUG_ON(!sgt->nents); + if (!sgt->nents) { + pr_err("sg table has invalid number of entries 0x%p.\n", + sgt); + return -EIO; + } } req = xdma_init_request(sgt, ep_addr); @@ -3021,25 +3848,39 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, goto unmap_sgl; } + //used when doing completion. + req->cb = cb; + cb->req = req; dbg_tfr("%s, len %u sg cnt %u.\n", engine->name, req->total_len, req->sw_desc_cnt); sg = sgt->sgl; nents = req->sw_desc_cnt; while (nents) { - unsigned long flags; + struct xdma_transfer *xfer; /* one transfer at a time */ - spin_lock(&engine->desc_lock); - - /* build transfer */ - rv = transfer_init(engine, req); + xfer = &req->tfer[tfer_idx]; + /* build transfer */ + rv = transfer_init(engine, req, xfer); if (rv < 0) { - spin_unlock(&engine->desc_lock); - goto unmap_sgl; + pr_info("transfer_init failed\n"); + + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, + sgt->orig_nents, dir); + sgt->nents = 0; + } + + /* Transfer failed return BUSY */ + if (cb->io_done) + cb->io_done((unsigned long)cb, -EBUSY); + + goto rel_req; } - xfer = &req->xfer; + + xfer->cb = cb; if (!dma_mapped) xfer->flags = XFER_FLAG_NEED_UNMAP; @@ -3051,9 +3892,9 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, xfer->sgt = sgt; } - dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", - xfer->len, req->ep_addr, done, req->sw_desc_idx, - req->sw_desc_cnt); + dbg_tfr("xfer %p, len %u, ep 0x%llx, sg %u/%u. nents = %d\n", + xfer, xfer->len, req->ep_addr, req->sw_desc_idx, + req->sw_desc_cnt, nents); #ifdef __LIBXDMA_DEBUG__ transfer_dump(xfer); @@ -3061,78 +3902,17 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, rv = transfer_queue(engine, xfer); if (rv < 0) { - spin_unlock(&engine->desc_lock); pr_info("unable to submit %s, %d.\n", engine->name, rv); goto unmap_sgl; } - /* - * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected + /* use multiple transfers per request if we could not fit all + * data within single descriptor chain. */ - if (poll_mode) { - unsigned int desc_count; - - spin_lock_irqsave(&engine->lock, flags); - desc_count = xfer->desc_num; - spin_unlock_irqrestore(&engine->lock, flags); - - dbg_tfr("%s poll desc_count=%d\n", - engine->name, desc_count); - rv = engine_service_poll(engine, desc_count); - - } else { - rv = wait_event_interruptible_timeout(xfer->wq, - (xfer->state != TRANSFER_STATE_SUBMITTED), - msecs_to_jiffies(timeout_ms)); - } - - spin_lock_irqsave(&engine->lock, flags); - - switch(xfer->state) { - case TRANSFER_STATE_COMPLETED: - spin_unlock_irqrestore(&engine->lock, flags); - - dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", - xfer, xfer->len, req->ep_addr - xfer->len, done); - done += xfer->len; - rv = 0; - break; - case TRANSFER_STATE_FAILED: - pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", - xfer, xfer->len, req->ep_addr - xfer->len); - spin_unlock_irqrestore(&engine->lock, flags); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -EIO; - break; - default: - /* transfer can still be in-flight */ - pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", - xfer, xfer->len, xfer->state, req->ep_addr); - engine_status_read(engine, 0, 1); - //engine_status_dump(engine); - transfer_abort(engine, xfer); - - xdma_engine_stop(engine); - spin_unlock_irqrestore(&engine->lock, flags); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -ERESTARTSYS; - break; - } - - transfer_destroy(xdev, xfer); - spin_unlock(&engine->desc_lock); + tfer_idx++; + } - if (rv < 0) - goto unmap_sgl; - } /* while (sg) */ + return -EIOCBQUEUED; unmap_sgl: if (!dma_mapped && sgt->nents) { @@ -3140,42 +3920,55 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, sgt->nents = 0; } +rel_req: if (req) xdma_request_free(req); - if (rv < 0) - return rv; - - return done; + return rv; } -EXPORT_SYMBOL_GPL(xdma_xfer_submit); +EXPORT_SYMBOL_GPL(xdma_xfer_submit_nowait); + int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) { - u8 *buffer_virt; - u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ - dma_addr_t buffer_bus; /* bus address */ + u32 max_consistent_size = XDMA_PERF_NUM_DESC * 32 * 1024; /* 4MB */ struct xdma_transfer *transfer; u64 ep_addr = 0; - int num_desc_in_a_loop = 128; + int num_desc_in_a_loop = XDMA_PERF_NUM_DESC; int size_in_desc = engine->xdma_perf->transfer_size; int size = size_in_desc * num_desc_in_a_loop; int i; + int rv = -ENOMEM; + unsigned char free_desc = 0; - BUG_ON(size_in_desc > max_consistent_size); + if (size_in_desc > max_consistent_size) { + pr_err("%s max consistent size %d is more than supported %d\n", + engine->name, size_in_desc, max_consistent_size); + return -EINVAL; + } if (size > max_consistent_size) { size = max_consistent_size; num_desc_in_a_loop = size / size_in_desc; } - buffer_virt = dma_alloc_coherent(&xdev->pdev->dev, size, - &buffer_bus, GFP_KERNEL); + engine->perf_buf_virt = dma_alloc_coherent(&xdev->pdev->dev, + size_in_desc, + &engine->perf_buf_bus, + GFP_KERNEL); + if (!engine->perf_buf_virt) { + pr_err("dev %s, %s DMA allocation OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + return rv; + } /* allocate transfer data structure */ transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); - BUG_ON(!transfer); - + if (!transfer) { + pr_err("dev %s, %s transfer request OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_engine_transfer; + } /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ transfer->dir = engine->dir; /* set number of descriptors */ @@ -3183,49 +3976,88 @@ int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) /* allocate descriptor list */ if (!engine->desc) { - engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + engine->desc = dma_alloc_coherent( + &xdev->pdev->dev, num_desc_in_a_loop * sizeof(struct xdma_desc), &engine->desc_bus, GFP_KERNEL); - BUG_ON(!engine->desc); + if (!engine->desc) { + pr_err("%s DMA memory allocation for descriptors failed\n", + engine->name); + goto err_engine_desc; + } dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", - dev_name(&xdev->pdev->dev), engine->name, - engine->desc, engine->desc_bus); + dev_name(&xdev->pdev->dev), engine->name, engine->desc, + engine->desc_bus); + free_desc = 1; } transfer->desc_virt = engine->desc; transfer->desc_bus = engine->desc_bus; - transfer_desc_init(transfer, transfer->desc_num); + rv = transfer_desc_init(transfer, transfer->desc_num); + if (rv < 0) { + pr_err("Failed to initialize descriptors\n"); + goto err_dma_desc; + } dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); for (i = 0; i < transfer->desc_num; i++) { struct xdma_desc *desc = transfer->desc_virt + i; - dma_addr_t rc_bus_addr = buffer_bus + size_in_desc * i; + dma_addr_t rc_bus_addr = engine->perf_buf_bus; /* fill in descriptor entry with transfer details */ xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, - engine->dir); + engine->dir); } /* stop engine and request interrupt on last descriptor */ - xdma_desc_control_set(transfer->desc_virt, 0); + rv = xdma_desc_control_set(transfer->desc_virt, 0); + if (rv < 0) { + pr_err("Failed to set desc control\n"); + goto err_dma_desc; + } /* create a linked loop */ xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, - transfer->desc_virt, transfer->desc_bus); + transfer->desc_virt, transfer->desc_bus); transfer->cyclic = 1; /* initialize wait queue */ +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + init_swait_queue_head(&transfer->wq); +#else init_waitqueue_head(&transfer->wq); +#endif - //printk("=== Descriptor print for PERF \n"); + //printk("=== Descriptor print for PERF\n"); //transfer_dump(transfer); dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", - engine->dir ? "write (to dev)" : "read (from dev)"); - transfer_queue(engine, transfer); + engine->dir ? "write (to dev)" : "read (from dev)"); + rv = transfer_queue(engine, transfer); + if (rv < 0) { + pr_err("Failed to queue transfer\n"); + goto err_dma_desc; + } return 0; +err_dma_desc: + if (free_desc && engine->desc) + dma_free_coherent(&xdev->pdev->dev, + num_desc_in_a_loop * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); + engine->desc = NULL; +err_engine_desc: + if (transfer) + list_del(&transfer->entry); + kfree(transfer); + transfer = NULL; +err_engine_transfer: + if (engine->perf_buf_virt) + dma_free_coherent(&xdev->pdev->dev, size_in_desc, + engine->perf_buf_virt, engine->perf_buf_bus); + engine->perf_buf_virt = NULL; + return rv; } EXPORT_SYMBOL_GPL(xdma_performance_submit); @@ -3235,7 +4067,10 @@ static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) struct xdma_dev *xdev; struct xdma_engine *engine; - BUG_ON(!pdev); + if (!pdev) { + pr_err("Invalid pdev\n"); + return NULL; + } /* allocate zeroed device book keeping structure */ xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); @@ -3267,19 +4102,29 @@ static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) engine = xdev->engine_h2c; for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { spin_lock_init(&engine->lock); - spin_lock_init(&engine->desc_lock); + mutex_init(&engine->desc_lock); INIT_LIST_HEAD(&engine->transfer_list); +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + init_swait_queue_head(&engine->shutdown_wq); + init_swait_queue_head(&engine->xdma_perf_wq); +#else init_waitqueue_head(&engine->shutdown_wq); init_waitqueue_head(&engine->xdma_perf_wq); +#endif } engine = xdev->engine_c2h; for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { spin_lock_init(&engine->lock); - spin_lock_init(&engine->desc_lock); + mutex_init(&engine->desc_lock); INIT_LIST_HEAD(&engine->transfer_list); +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + init_swait_queue_head(&engine->shutdown_wq); + init_swait_queue_head(&engine->xdma_perf_wq); +#else init_waitqueue_head(&engine->shutdown_wq); init_waitqueue_head(&engine->xdma_perf_wq); +#endif } return xdev; @@ -3289,8 +4134,15 @@ static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) { int rv; - BUG_ON(!xdev); - BUG_ON(!pdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } + + if (!pdev) { + pr_err("Invalid pdev\n"); + return -EINVAL; + } dbg_init("pci_request_regions()\n"); rv = pci_request_regions(pdev, xdev->mod_name); @@ -3308,7 +4160,10 @@ static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) static int set_dma_mask(struct pci_dev *pdev) { - BUG_ON(!pdev); + if (!pdev) { + pr_err("Invalid pdev\n"); + return -EINVAL; + } dbg_init("sizeof(dma_addr_t) == %ld\n", sizeof(dma_addr_t)); /* 64-bit addressing capability for XDMA? */ @@ -3334,22 +4189,28 @@ static int set_dma_mask(struct pci_dev *pdev) return 0; } -static u32 get_engine_channel_id(struct engine_regs *regs) +static int get_engine_channel_id(struct engine_regs *regs) { - u32 value; + int value; - BUG_ON(!regs); + if (!regs) { + pr_err("Invalid engine registers\n"); + return -EINVAL; + } value = read_register(®s->identifier); return (value & 0x00000f00U) >> 8; } -static u32 get_engine_id(struct engine_regs *regs) +static int get_engine_id(struct engine_regs *regs) { - u32 value; + int value; - BUG_ON(!regs); + if (!regs) { + pr_err("Invalid engine registers\n"); + return -EINVAL; + } value = read_register(®s->identifier); return (value & 0xffff0000U) >> 16; @@ -3359,15 +4220,21 @@ static void remove_engines(struct xdma_dev *xdev) { struct xdma_engine *engine; int i; + int rv; - BUG_ON(!xdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return; + } /* iterate over channels */ for (i = 0; i < xdev->h2c_channel_max; i++) { engine = &xdev->engine_h2c[i]; if (engine->magic == MAGIC_ENGINE) { dbg_sg("Remove %s, %d", engine->name, i); - engine_destroy(xdev, engine); + rv = engine_destroy(xdev, engine); + if (rv < 0) + pr_err("Failed to destroy H2C engine %d\n", i); dbg_sg("%s, %d removed", engine->name, i); } } @@ -3376,14 +4243,16 @@ static void remove_engines(struct xdma_dev *xdev) engine = &xdev->engine_c2h[i]; if (engine->magic == MAGIC_ENGINE) { dbg_sg("Remove %s, %d", engine->name, i); - engine_destroy(xdev, engine); + rv = engine_destroy(xdev, engine); + if (rv < 0) + pr_err("Failed to destroy C2H engine %d\n", i); dbg_sg("%s, %d removed", engine->name, i); } } } static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, - int channel) + int channel) { struct engine_regs *regs; int offset = channel * CHANNEL_SPACING; @@ -3395,7 +4264,8 @@ static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, /* register offset for the engine */ /* read channels at 0x0000, write channels at 0x1000, - * channels at 0x100 interval */ + * channels at 0x100 interval + */ if (dir == DMA_TO_DEVICE) { engine_id_expected = XDMA_ID_H2C; engine = &xdev->engine_h2c[channel]; @@ -3410,24 +4280,23 @@ static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, channel_id = get_engine_channel_id(regs); if ((engine_id != engine_id_expected) || (channel_id != channel)) { - dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," - "exp 0x%x,0x%x, SKIP.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", - channel, offset, engine_id, channel_id, - engine_id_expected, channel_id != channel); + dbg_init( + "%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x,exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, offset, + engine_id, channel_id, engine_id_expected, + channel_id != channel); return -EINVAL; } dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, - offset, engine_id, channel_id); + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, offset, + engine_id, channel_id); /* allocate and initialize engine */ rv = engine_init(engine, xdev, offset, dir, channel); if (rv != 0) { pr_info("failed to create AXI %s %d engine.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", - channel); + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel); return rv; } @@ -3439,7 +4308,10 @@ static int probe_engines(struct xdma_dev *xdev) int i; int rv = 0; - BUG_ON(!xdev); + if (!xdev) { + pr_err("Invalid xdev\n"); + return -EINVAL; + } /* iterate over channels */ for (i = 0; i < xdev->h2c_channel_max; i++) { @@ -3459,13 +4331,13 @@ static int probe_engines(struct xdma_dev *xdev) return 0; } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) -static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +#if KERNEL_VERSION(3, 5, 0) <= LINUX_VERSION_CODE +static void pci_enable_capability(struct pci_dev *pdev, int cap) { - pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, cap); } #else -static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +static void pci_enable_capability(struct pci_dev *pdev, int cap) { u16 v; int pos; @@ -3473,52 +4345,14 @@ static void pci_enable_relaxed_ordering(struct pci_dev *pdev) pos = pci_pcie_cap(pdev); if (pos > 0) { pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v); - v |= PCI_EXP_DEVCTL_RELAX_EN; + v |= cap; pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v); } } #endif -static void pci_check_extended_tag(struct xdma_dev *xdev, struct pci_dev *pdev) -{ - u16 cap; - u32 v; - void *__iomem reg; - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) - pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); -#else - int pos; - - pos = pci_pcie_cap(pdev); - if (pos > 0) - pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &cap); - else { - pr_info("pdev 0x%p, unable to access pcie cap.\n", pdev); - return; - } -#endif - - if ((cap & PCI_EXP_DEVCTL_EXT_TAG)) - return; - - /* extended tag not enabled */ - pr_info("0x%p EXT_TAG disabled.\n", pdev); - - if (xdev->config_bar_idx < 0) { - pr_info("pdev 0x%p, xdev 0x%p, config bar UNKNOWN.\n", - pdev, xdev); - return; - } - - reg = xdev->bar[xdev->config_bar_idx] + XDMA_OFS_CONFIG + 0x4C; - v = read_register(reg); - v = (v & 0xFF) | (((u32)32) << 8); - write_register(v, reg, XDMA_OFS_CONFIG + 0x4C); -} - void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, - int *h2c_channel_max, int *c2h_channel_max) + int *h2c_channel_max, int *c2h_channel_max) { struct xdma_dev *xdev = NULL; int rv = 0; @@ -3540,12 +4374,12 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) xdev->user_max = MAX_USER_IRQ; if (xdev->h2c_channel_max == 0 || - xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; if (xdev->c2h_channel_max == 0 || - xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; - + rv = pci_enable_device(pdev); if (rv) { dbg_init("pci_enable_device() failed, %d.\n", rv); @@ -3556,9 +4390,10 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, pci_check_intr_pend(pdev); /* enable relaxed ordering */ - pci_enable_relaxed_ordering(pdev); + pci_enable_capability(pdev, PCI_EXP_DEVCTL_RELAX_EN); - pci_check_extended_tag(xdev, pdev); + /* enable extended tag */ + pci_enable_capability(pdev, PCI_EXP_DEVCTL_EXT_TAG); /* force MRRS to be 512 */ rv = pcie_set_readrq(pdev, 512); @@ -3645,11 +4480,11 @@ void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) return; - dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", - pdev, xdev); + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", pdev, + xdev); if (xdev->pdev != pdev) { dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", - (unsigned long)xdev->pdev, (unsigned long)pdev); + (unsigned long)xdev->pdev, (unsigned long)pdev); } channel_interrupts_disable(xdev, ~0); @@ -3683,6 +4518,7 @@ void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; struct xdma_engine *engine; int i; + int rv; if (!dev_hndl) return; @@ -3690,26 +4526,29 @@ void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) return; -pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); /* wait for all engines to be idle */ - for (i = 0; i < xdev->h2c_channel_max; i++) { + for (i = 0; i < xdev->h2c_channel_max; i++) { unsigned long flags; engine = &xdev->engine_h2c[i]; - + if (engine->magic == MAGIC_ENGINE) { spin_lock_irqsave(&engine->lock, flags); engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; - xdma_engine_stop(engine); - engine->running = 0; + rv = xdma_engine_stop(engine); + if (rv < 0) + pr_err("Failed to stop engine\n"); + else + engine->running = 0; spin_unlock_irqrestore(&engine->lock, flags); } } - for (i = 0; i < xdev->c2h_channel_max; i++) { + for (i = 0; i < xdev->c2h_channel_max; i++) { unsigned long flags; engine = &xdev->engine_c2h[i]; @@ -3717,8 +4556,11 @@ pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); spin_lock_irqsave(&engine->lock, flags); engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; - xdma_engine_stop(engine); - engine->running = 0; + rv = xdma_engine_stop(engine); + if (rv < 0) + pr_err("Failed to stop engine\n"); + else + engine->running = 0; spin_unlock_irqrestore(&engine->lock, flags); } } @@ -3746,9 +4588,9 @@ void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) return; -pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); - for (i = 0; i < xdev->h2c_channel_max; i++) { + for (i = 0; i < xdev->h2c_channel_max; i++) { engine = &xdev->engine_h2c[i]; if (engine->magic == MAGIC_ENGINE) { engine_init_regs(engine); @@ -3758,7 +4600,7 @@ pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); } } - for (i = 0; i < xdev->c2h_channel_max; i++) { + for (i = 0; i < xdev->c2h_channel_max; i++) { engine = &xdev->engine_c2h[i]; if (engine->magic == MAGIC_ENGINE) { engine_init_regs(engine); @@ -3776,9 +4618,9 @@ pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); user_interrupts_enable(xdev, xdev->mask_irq_user); read_interrupts(xdev); } - + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); -pr_info("xdev 0x%p, done.\n", xdev); + pr_info("xdev 0x%p, done.\n", xdev); } EXPORT_SYMBOL_GPL(xdma_device_online); @@ -3798,7 +4640,7 @@ int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) EXPORT_SYMBOL_GPL(xdma_device_restart); int xdma_user_isr_register(void *dev_hndl, unsigned int mask, - irq_handler_t handler, void *dev) + irq_handler_t handler, void *dev) { struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; int i; @@ -3852,7 +4694,7 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) return -EINVAL; - + xdev->mask_irq_user &= ~mask; user_interrupts_disable(xdev, mask); read_interrupts(xdev); @@ -3864,13 +4706,13 @@ EXPORT_SYMBOL_GPL(xdma_user_isr_disable); #ifdef __LIBXDMA_MOD__ static int __init xdma_base_init(void) { - printk(KERN_INFO "%s", version); + pr_info("%s", version); return 0; } static void __exit xdma_base_exit(void) { - return; + pr_info("%s", __func__); } module_init(xdma_base_init); @@ -3881,25 +4723,38 @@ static void xdma_transfer_cyclic(struct xdma_transfer *transfer) { /* link last descriptor to first descriptor */ xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, - transfer->desc_virt, transfer->desc_bus); + transfer->desc_virt, transfer->desc_bus); /* remember transfer is cyclic */ transfer->cyclic = 1; } static int transfer_monitor_cyclic(struct xdma_engine *engine, - struct xdma_transfer *transfer, int timeout_ms) + struct xdma_transfer *transfer, + int timeout_ms) { struct xdma_result *result; int rc = 0; - BUG_ON(!engine); - BUG_ON(!transfer); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (!transfer) { + pr_err("%s: xfer empty.\n", engine->name); + return -EINVAL; + } result = engine->cyclic_result; - BUG_ON(!result); + if (!result) { + pr_err("%s Cyclic transfer resources not available.\n", + engine->name); + return -EINVAL; + } if (poll_mode) { - int i ; + int i; + for (i = 0; i < 5; i++) { rc = engine_service_poll(engine, 0); if (rc) { @@ -3907,33 +4762,46 @@ static int transfer_monitor_cyclic(struct xdma_engine *engine, engine->name, rc); rc = -ERESTARTSYS; } - if (result[engine->rx_head].status) - return 0; + if (result[engine->rx_head].status) { + rc = 0; + break; + } } } else { - if (enable_credit_mp){ + if (enable_credit_mp) { dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", engine->name, engine->rx_head, engine->rx_tail); - rc = wait_event_interruptible_timeout( transfer->wq, - (engine->rx_head!=engine->rx_tail || - engine->rx_overrun), - msecs_to_jiffies(timeout_ms)); + + rc = xlx_wait_event_interruptible_timeout( + transfer->wq, + (engine->rx_head != engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", - engine->name, rc, engine->rx_head, + engine->name, rc, engine->rx_head, engine->rx_tail, engine->rx_overrun); } else { - rc = wait_event_interruptible_timeout( transfer->wq, - engine->eop_found, - msecs_to_jiffies(timeout_ms)); + rc = xlx_wait_event_interruptible_timeout( + transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, eop_found %d.\n", engine->name, rc, engine->eop_found); } + /* condition evaluated to false after the timeout elapsed */ + if (rc == 0) + rc = -ETIME; + /* condition evaluated to true */ + else if (rc > 0) + rc = 0; } - return 0; + return rc; } -struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +static struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) { struct scatterlist *sg = sgt->sgl; int i; @@ -3951,42 +4819,48 @@ struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) } static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, - int head, char __user *buf, size_t count) + int head, char __user *buf, size_t count) { struct scatterlist *sg; int more = pkt_length; - BUG_ON(!engine); - BUG_ON(!buf); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (!buf) { + pr_err("%s invalid user buffer pointer\n", engine->name); + return -EINVAL; + } - dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", - engine->name, pkt_length, head, engine->user_buffer_index); + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", engine->name, + pkt_length, head, engine->user_buffer_index); sg = sglist_index(&engine->cyclic_sgt, head); if (!sg) { - pr_info("%s, head %d OOR, sgl %u.\n", - engine->name, head, engine->cyclic_sgt.orig_nents); + pr_info("%s, head %d OOR, sgl %u.\n", engine->name, head, + engine->cyclic_sgt.orig_nents); return -EIO; } /* EOP found? Transfer anything from head to EOP */ while (more) { - unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; unsigned int blen = count - engine->user_buffer_index; int rv; if (copy > blen) copy = blen; - dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", - engine->name, head, sg, copy, - engine->user_buffer_index); + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", engine->name, + head, sg, copy, engine->user_buffer_index); rv = copy_to_user(&buf[engine->user_buffer_index], - page_address(sg_page(sg)), copy); + page_address(sg_page(sg)), copy); if (rv) { - pr_info("%s copy_to_user %u failed %d\n", - engine->name, copy, rv); + pr_info("%s copy_to_user %u failed %d\n", engine->name, + copy, rv); return -EIO; } @@ -3997,7 +4871,7 @@ static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, /* user buffer used up */ break; } - + head++; if (head >= CYCLIC_RX_PAGES_MAX) { head = 0; @@ -4021,9 +4895,17 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, int num_credit = 0; unsigned long flags; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } result = engine->cyclic_result; - BUG_ON(!result); + + if (!result) { + pr_err("%s Cyclic transfer resources not available.\n", + engine->name); + return -EINVAL; + } spin_lock_irqsave(&engine->lock, flags); @@ -4031,12 +4913,12 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, head = engine->rx_head; /* iterate over newly received results */ - while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { - - WARN_ON(result[engine->rx_head].status==0); + while ((engine->rx_head != engine->rx_tail || engine->rx_overrun) && + pkt_length < (int)count) { + WARN_ON(result[engine->rx_head].status == 0); dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", - engine->name, engine->rx_head, + engine->name, engine->rx_head, result[engine->rx_head].status, result[engine->rx_head].length); @@ -4051,24 +4933,23 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, result[engine->rx_head].length, PAGE_SIZE); fault = 1; } else if (result[engine->rx_head].length == 0) { - pr_info("%s, result[%d].length 0x%x.\n", - engine->name, engine->rx_head, + pr_info("%s, result[%d].length 0x%x.\n", engine->name, + engine->rx_head, result[engine->rx_head].length); fault = 1; /* valid result */ } else { pkt_length += result[engine->rx_head].length; - num_credit++; + num_credit++; /* seen eop? */ //if (result[engine->rx_head].status & RX_STATUS_EOP) - if (result[engine->rx_head].status & RX_STATUS_EOP){ + if (result[engine->rx_head].status & RX_STATUS_EOP) { eop = 1; engine->eop_found = 1; } - dbg_tfr("%s, pkt_length=%d (%s)\n", - engine->name, pkt_length, - eop ? "with EOP" : "no EOP yet"); + dbg_tfr("%s, pkt_length=%d (%s)\n", engine->name, + pkt_length, eop ? "with EOP" : "no EOP yet"); } /* clear result */ result[engine->rx_head].status = 0; @@ -4077,21 +4958,20 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; /* stop processing if a fault/eop was detected */ - if (fault || eop){ + if (fault || eop) break; - } } spin_unlock_irqrestore(&engine->lock, flags); if (fault) return -EIO; - + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); - engine->rx_overrun = 0; + engine->rx_overrun = 0; /* if copy is successful, release credits */ - if(rc > 0) - write_register(num_credit,&engine->sgdma_regs->credits, 0); + if (rc > 0) + write_register(num_credit, &engine->sgdma_regs->credits, 0); return rc; } @@ -4104,14 +4984,25 @@ ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, int rc_len = 0; struct xdma_transfer *transfer; - BUG_ON(!engine); - BUG_ON(engine->magic != MAGIC_ENGINE); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + + if (engine->magic != MAGIC_ENGINE) { + pr_err("%s has invalid magic number %lx\n", engine->name, + engine->magic); + return -EINVAL; + } + + transfer = &engine->cyclic_req->tfer[0]; + if (!transfer) { + pr_err("Invalid DMA transfer\n"); + return -EINVAL; + } - transfer = &engine->cyclic_req->xfer; - BUG_ON(!transfer); + engine->user_buffer_index = 0; - engine->user_buffer_index = 0; - do { rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); if (rc < 0) @@ -4121,13 +5012,16 @@ ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, return rc; rc_len += rc; + if (rc_len >= (int)count) + return rc_len; + i++; if (i > 10) break; } while (!engine->eop_found); - if(enable_credit_mp) - engine->eop_found = 0; + //Always reset EOP found indication regardless of credit mechanims + engine->eop_found = 0; return rc_len; } @@ -4169,17 +5063,17 @@ static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, for (i = 0; i < npages; i++, sg = sg_next(sg)) { struct page *pg = alloc_page(GFP_KERNEL); - if (!pg) { + if (!pg) { pr_info("%d/%u, page OOM.\n", i, npages); goto err_out; } if (pdev) { - dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, - dir); - if (unlikely(pci_dma_mapping_error(pdev, bus))) { - pr_info("%d/%u, page 0x%p map err.\n", - i, npages, pg); + dma_addr_t bus = + pci_map_page(pdev, pg, 0, PAGE_SIZE, dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", i, + npages, pg); __free_page(pg); goto err_out; } @@ -4195,7 +5089,7 @@ static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, err_out: sgt_free_with_pages(sgt, dir, pdev); - return -ENOMEM; + return -ENOMEM; } int xdma_cyclic_transfer_setup(struct xdma_engine *engine) @@ -4207,13 +5101,19 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) int i; int rc; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } + xdev = engine->xdev; - BUG_ON(!xdev); + if (!xdev) { + pr_err("Invalid DMA devie\n"); + return -EINVAL; + } if (engine->cyclic_req) { - pr_info("%s: exclusive access already taken.\n", - engine->name); + dbg_tfr("%s: exclusive access already taken.\n", engine->name); return -EBUSY; } @@ -4225,10 +5125,10 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) engine->eop_found = 0; rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, - engine->dir, xdev->pdev); + engine->dir, xdev->pdev); if (rc < 0) { - pr_info("%s cyclic pages %u OOM.\n", - engine->name, CYCLIC_RX_PAGES_MAX); + pr_info("%s cyclic pages %u OOM.\n", engine->name, + CYCLIC_RX_PAGES_MAX); goto err_out; } @@ -4243,27 +5143,34 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) xdma_request_cb_dump(engine->cyclic_req); #endif - rc = transfer_init(engine, engine->cyclic_req); + xfer = &engine->cyclic_req->tfer[0]; + rc = transfer_init_cyclic(engine, engine->cyclic_req, xfer); if (rc < 0) goto err_out; - xfer = &engine->cyclic_req->xfer; - /* replace source addresses with result write-back addresses */ memset(engine->cyclic_result, 0, - CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); bus = engine->cyclic_result_bus; - for (i = 0; i < xfer->desc_num; i++) { + for (i = 0; i < xfer->desc_num; i++) { xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); - xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); - bus += sizeof(struct xdma_result); - } + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } /* set control of all descriptors */ - for (i = 0; i < xfer->desc_num; i++) { - xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); - xdma_desc_control_set(xfer->desc_virt + i, - XDMA_DESC_EOP | XDMA_DESC_COMPLETED); - } + for (i = 0; i < xfer->desc_num; i++) { + rc = xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + if (rc < 0) { + pr_err("Failed to clear desc control\n"); + goto err_out; + } + rc = xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + if (rc < 0) { + pr_err("Failed to set desc control\n"); + goto err_out; + } + } /* make this a cyclic transfer */ xdma_transfer_cyclic(xfer); @@ -4272,28 +5179,34 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) transfer_dump(xfer); #endif - if(enable_credit_mp){ + if (enable_credit_mp) { //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); write_register(128, &engine->sgdma_regs->credits, 0); } - spin_unlock_irqrestore(&engine->lock, flags); - /* start cyclic transfer */ - transfer_queue(engine, xfer); + rc = transfer_queue(engine, xfer); + if (rc < 0) { + pr_err("Failed to queue transfer\n"); + goto err_out; + } + + xfer->last_in_request = 1; + + spin_unlock_irqrestore(&engine->lock, flags); return 0; /* unwind on errors */ err_out: if (engine->cyclic_req) { - xdma_request_free(engine->cyclic_req); + xdma_request_free(engine->cyclic_req); engine->cyclic_req = NULL; } - + if (engine->cyclic_sgt.orig_nents) { sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, - xdev->pdev); + xdev->pdev); engine->cyclic_sgt.orig_nents = 0; engine->cyclic_sgt.nents = 0; engine->cyclic_sgt.sgl = NULL; @@ -4304,42 +5217,63 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) return rc; } - static int cyclic_shutdown_polled(struct xdma_engine *engine) { - BUG_ON(!engine); + int rv; + + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } spin_lock(&engine->lock); dbg_tfr("Polling for shutdown completion\n"); do { - engine_status_read(engine, 1, 0); + rv = engine_status_read(engine, 1, 0); + if (rv < 0) { + pr_err("Failed to read engine status\n"); + goto failure; + } schedule(); } while (engine->status & XDMA_STAT_BUSY); if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { dbg_tfr("Engine has stopped\n"); - if (!list_empty(&engine->transfer_list)) - engine_transfer_dequeue(engine); + if (!list_empty(&engine->transfer_list)) { + rv = engine_transfer_dequeue(engine); + if (rv < 0) { + pr_err("Failed to dequeue transfer\n"); + goto failure; + } + } - engine_service_shutdown(engine); + rv = engine_service_shutdown(engine); + if (rv < 0) { + pr_err("Failed to shutdown engine\n"); + goto failure; + } } - +failure: dbg_tfr("Shutdown completion polling done\n"); spin_unlock(&engine->lock); - return 0; + return rv; } static int cyclic_shutdown_interrupt(struct xdma_engine *engine) { int rc; - BUG_ON(!engine); + if (!engine) { + pr_err("dma engine NULL\n"); + return -EINVAL; + } - rc = wait_event_interruptible_timeout(engine->shutdown_wq, - !engine->running, msecs_to_jiffies(10000)); + rc = xlx_wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, + msecs_to_jiffies(10000)); #if 0 if (rc) { @@ -4364,36 +5298,45 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) unsigned long flags; transfer = engine_cyclic_stop(engine); + if (transfer == NULL) { + pr_err("Failed to stop cyclic engine\n"); + return -EINVAL; + } spin_lock_irqsave(&engine->lock, flags); if (transfer) { dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); - if (transfer != &engine->cyclic_req->xfer) { + if (transfer != &engine->cyclic_req->tfer[0]) { pr_info("%s unexpected transfer 0x%p/0x%p\n", engine->name, transfer, - &engine->cyclic_req->xfer); + &engine->cyclic_req->tfer[0]); } } /* allow engine to be serviced after stop request */ spin_unlock_irqrestore(&engine->lock, flags); /* wait for engine to be no longer running */ - if (poll_mode) + if (poll_mode) rc = cyclic_shutdown_polled(engine); else rc = cyclic_shutdown_interrupt(engine); + if (rc < 0) { + pr_err("Failed to shutdown cyclic transfers\n"); + return rc; + } + /* obtain spin lock to atomically remove resources */ spin_lock_irqsave(&engine->lock, flags); if (engine->cyclic_req) { - xdma_request_free(engine->cyclic_req); + xdma_request_free(engine->cyclic_req); engine->cyclic_req = NULL; } if (engine->cyclic_sgt.orig_nents) { sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, - xdev->pdev); + xdev->pdev); engine->cyclic_sgt.orig_nents = 0; engine->cyclic_sgt.nents = 0; engine->cyclic_sgt.sgl = NULL; @@ -4407,25 +5350,26 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) { int rv; - unsigned long dst; + unsigned long dst; u32 w = XDMA_CTRL_NON_INCR_ADDR; dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); rv = get_user(dst, (int __user *)arg); - if (rv == 0) { + if (rv == 0) { engine->non_incr_addr = !!dst; if (engine->non_incr_addr) - write_register(w, &engine->regs->control_w1s, + write_register( + w, &engine->regs->control_w1s, (unsigned long)(&engine->regs->control_w1s) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); else - write_register(w, &engine->regs->control_w1c, - (unsigned long)(&engine->regs->control_w1c) - - (unsigned long)(&engine->regs)); + write_register( + w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); } engine_alignments(engine); return rv; } - diff --git a/sdk/linux_kernel_drivers/xdma/libxdma.h b/sdk/linux_kernel_drivers/xdma/libxdma.h index 07d016c28..9f889cd9f 100755 --- a/sdk/linux_kernel_drivers/xdma/libxdma.h +++ b/sdk/linux_kernel_drivers/xdma/libxdma.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef XDMA_LIB_H #define XDMA_LIB_H @@ -35,6 +36,14 @@ #include #include #include +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE +#include +#endif +/* + * if the config bar is fixed, the driver does not neeed to search through + * all of the bars + */ +//#define XDMA_CONFIG_BAR_NUM 1 /* Switch debug printing on/off */ #define XDMA_DEBUG 0 @@ -53,9 +62,9 @@ * interrupts per engine, rad2_vul.sv:237 * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), */ -#define XDMA_ENG_IRQ_NUM (1) -#define MAX_EXTRA_ADJ (15) -#define RX_STATUS_EOP (1) +#define XDMA_ENG_IRQ_NUM (1) +#define MAX_EXTRA_ADJ (0x3F) +#define RX_STATUS_EOP (1) /* Target internal components on XDMA control BAR */ #define XDMA_OFS_INT_CTRL (0x2000UL) @@ -65,7 +74,7 @@ #define XDMA_TRANSFER_MAX_DESC (2048) /* maximum size of a single DMA transfer descriptor */ -#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_BITS 28 #define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1) /* bits of the SG DMA control register */ @@ -79,6 +88,7 @@ #define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) #define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) #define XDMA_CTRL_POLL_MODE_WB (1UL << 26) +#define XDMA_CTRL_STM_MODE_WB (1UL << 27) /* bits of the SG DMA status register */ #define XDMA_STAT_BUSY (1UL << 0) @@ -134,7 +144,7 @@ /* all combined */ #define XDMA_STAT_H2C_ERR_MASK \ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ - XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) #define XDMA_STAT_C2H_ERR_MASK \ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ @@ -157,7 +167,7 @@ #define XDMA_ID_C2H 0x1fc1U /* for C2H AXI-ST mode */ -#define CYCLIC_RX_PAGES_MAX 256 +#define CYCLIC_RX_PAGES_MAX 256 #define LS_BYTE_MASK 0x000000FFUL @@ -243,6 +253,23 @@ enum dev_capabilities { /* SECTION: Structure definitions */ +struct xdma_io_cb { + void __user *buf; + size_t len; + void *private; + unsigned int pages_nr; + struct sg_table sgt; + struct page **pages; + /** total data size */ + unsigned int count; + /** MM only, DDR/BRAM memory addr */ + u64 ep_addr; + /** write: if write to the device */ + struct xdma_request_cb *req; + u8 write:1; + void (*io_done)(unsigned long cb_hndl, int err); +}; + struct config_regs { u32 identifier; u32 reserved_1[4]; @@ -388,11 +415,18 @@ struct sw_desc { struct xdma_transfer { struct list_head entry; /* queue of non-completed transfers */ struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ + struct xdma_result *res_virt; /* virt addr of result, c2h streaming */ + dma_addr_t res_bus; /* bus addr for result descriptors */ dma_addr_t desc_bus; /* bus addr of the first descriptor */ int desc_adjacent; /* adjacent descriptors at desc_bus */ int desc_num; /* number of descriptors in transfer */ + int desc_index; /* index for 1st desc. in transfer */ enum dma_data_direction dir; +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + struct swait_queue_head wq; +#else wait_queue_head_t wq; /* wait queue for transfer completion */ +#endif enum transfer_state state; /* state of the transfer */ unsigned int flags; @@ -401,6 +435,7 @@ struct xdma_transfer { int last_in_request; /* flag if last within request */ unsigned int len; struct sg_table *sgt; + struct xdma_io_cb *cb; }; struct xdma_request_cb { @@ -408,7 +443,10 @@ struct xdma_request_cb { unsigned int total_len; u64 ep_addr; - struct xdma_transfer xfer; + /* Use two transfers in case single request needs to be split */ + struct xdma_transfer tfer[2]; + + struct xdma_io_cb *cb; unsigned int sw_desc_idx; unsigned int sw_desc_cnt; @@ -442,7 +480,8 @@ struct xdma_engine { int max_extra_adj; /* descriptor prefetch capability */ int desc_dequeued; /* num descriptors of completed transfers */ u32 status; /* last known status of device */ - u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */ + /* only used for MSIX mode to store per-engine interrupt mask value */ + u32 interrupt_enable_mask_value; /* Transfer list management */ struct list_head transfer_list; /* queue of transfers */ @@ -450,10 +489,13 @@ struct xdma_engine { /* Members applicable to AXI-ST C2H (cyclic) transfers */ struct xdma_result *cyclic_result; dma_addr_t cyclic_result_bus; /* bus addr for transfer */ - struct xdma_request_cb *cyclic_req; - struct sg_table cyclic_sgt; - u8 eop_found; /* used only for cyclic(rx:c2h) */ + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 *perf_buf_virt; + dma_addr_t perf_buf_bus; /* bus address */ + u8 eop_found; /* used only for cyclic(rx:c2h) */ + int eop_count; int rx_tail; /* follows the HW */ int rx_head; /* where the SW reads from */ int rx_overrun; /* flag if overrun occured */ @@ -466,20 +508,37 @@ struct xdma_engine { dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ /* Members associated with interrupt mode support */ +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + struct swait_queue_head shutdown_wq; +#else wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ +#endif spinlock_t lock; /* protects concurrent access */ int prev_cpu; /* remember CPU# of (last) locker */ int msix_irq_line; /* MSI-X vector for this engine */ u32 irq_bitmask; /* IRQ bit mask for this engine */ struct work_struct work; /* Work queue for interrupt handling */ - spinlock_t desc_lock; /* protects concurrent access */ + struct mutex desc_lock; /* protects concurrent access */ dma_addr_t desc_bus; struct xdma_desc *desc; + int desc_idx; /* current descriptor index */ + int desc_used; /* total descriptors used */ /* for performance test support */ struct xdma_performance_ioctl *xdma_perf; /* perf test control */ +#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE + struct swait_queue_head xdma_perf_wq; +#else wait_queue_head_t xdma_perf_wq; /* Perf test sync */ +#endif + + struct xdma_kthread *cmplthp; + /* completion status thread list for the queue */ + struct list_head cmplthp_list; + /* pending work thread list */ + /* cpu attached to intr_work */ + unsigned int intr_work_cpu; }; struct xdma_user_irq { @@ -490,14 +549,14 @@ struct xdma_user_irq { wait_queue_head_t events_wq; /* wait queue to sync waiting threads */ irq_handler_t handler; - void *dev; + void *dev; }; /* XDMA PCIe device specific book-keeping */ #define XDEV_FLAG_OFFLINE 0x1 struct xdma_dev { struct list_head list_head; - struct list_head rcu_node; + struct list_head rcu_node; unsigned long magic; /* structure ID for sanity checks */ struct pci_dev *pdev; /* pci device struct from probe() */ @@ -509,7 +568,7 @@ struct xdma_dev { unsigned int flags; /* PCIe BAR management */ - void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ + void __iomem *bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ int user_bar_idx; /* BAR index of user logic */ int config_bar_idx; /* BAR index of XDMA config logic */ int bypass_bar_idx; /* BAR index of XDMA bypass logic */ @@ -525,7 +584,7 @@ struct xdma_dev { int irq_line; /* flag if irq allocated successfully */ int msi_enabled; /* flag if msi was enabled for the device */ int msix_enabled; /* flag if msi-x was enabled for the device */ -#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) +#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE struct msix_entry entry[32]; /* msi-x vector/entry table */ #endif struct xdma_user_irq user_irq[16]; /* user IRQ management */ @@ -605,8 +664,8 @@ void get_perf_stats(struct xdma_engine *engine); int xdma_cyclic_transfer_setup(struct xdma_engine *engine); int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); -ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t, - int); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, + char __user *buf, size_t count, int timeout_ms); int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); - +int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count); #endif /* XDMA_LIB_H */ diff --git a/sdk/linux_kernel_drivers/xdma/libxdma_api.h b/sdk/linux_kernel_drivers/xdma/libxdma_api.h index bf043eb12..f652d7d9d 100644 --- a/sdk/linux_kernel_drivers/xdma/libxdma_api.h +++ b/sdk/linux_kernel_drivers/xdma/libxdma_api.h @@ -1,12 +1,24 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver + * Copyright(c) 2015 - 2020 Xilinx, Inc. * - * Copyright(c) Sidebranch. - * Copyright(c) Xilinx, Inc. + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + * + * The full GNU General Public License is included in this distribution in + * the file called "LICENSE". * * Karen Xie - * Leon Woestenberg * ******************************************************************************/ @@ -125,6 +137,14 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask); */ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, struct sg_table *sgt, bool dma_mapped, int timeout_ms); + +ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms); + + +ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms); + /////////////////////missing API//////////////////// diff --git a/sdk/linux_kernel_drivers/xdma/version.h b/sdk/linux_kernel_drivers/xdma/version.h index 64b91799f..094a285fd 100755 --- a/sdk/linux_kernel_drivers/xdma/version.h +++ b/sdk/linux_kernel_drivers/xdma/version.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,12 +21,13 @@ * Karen Xie * ******************************************************************************/ + #ifndef __XDMA_VERSION_H__ #define __XDMA_VERSION_H__ -#define DRV_MOD_MAJOR 2017 +#define DRV_MOD_MAJOR 2020 #define DRV_MOD_MINOR 1 -#define DRV_MOD_PATCHLEVEL 47 +#define DRV_MOD_PATCHLEVEL 06 #define DRV_MODULE_VERSION \ __stringify(DRV_MOD_MAJOR) "." \ diff --git a/sdk/linux_kernel_drivers/xdma/xdma_cdev.c b/sdk/linux_kernel_drivers/xdma/xdma_cdev.c index 8a3311618..3a540c82d 100644 --- a/sdk/linux_kernel_drivers/xdma/xdma_cdev.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_cdev.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,11 +21,14 @@ * Karen Xie * ******************************************************************************/ + #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include "xdma_cdev.h" -struct class *g_xdma_class; +static struct class *g_xdma_class; + +struct kmem_cache *cdev_cache; enum cdev_type { CHAR_USER, @@ -52,12 +55,12 @@ static const char * const devnode_names[] = { }; enum xpdev_flags_bits { - XDF_CDEV_USER, - XDF_CDEV_CTRL, - XDF_CDEV_XVC, - XDF_CDEV_EVENT, - XDF_CDEV_SG, - XDF_CDEV_BYPASS, + XDF_CDEV_USER, + XDF_CDEV_CTRL, + XDF_CDEV_XVC, + XDF_CDEV_EVENT, + XDF_CDEV_SG, + XDF_CDEV_BYPASS, }; static inline void xpdev_flag_set(struct xdma_pci_dev *xpdev, @@ -79,16 +82,18 @@ static inline int xpdev_flag_test(struct xdma_pci_dev *xpdev, } #ifdef __XDMA_SYSFS__ -ssize_t show_device_numbers(struct device *dev, struct device_attribute *attr, - char *buf) +ssize_t xdma_dev_instance_show(struct device *dev, + struct device_attribute *attr, + char *buf) { - struct xdma_pci_dev *xpdev = (struct xdma_pci_dev *)dev_get_drvdata(dev); + struct xdma_pci_dev *xpdev = + (struct xdma_pci_dev *)dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%d\t%d\n", xpdev->major, xpdev->xdev->idx); } -static DEVICE_ATTR(xdma_dev_instance, S_IRUGO, show_device_numbers, NULL); +static DEVICE_ATTR_RO(xdma_dev_instance); #endif static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type) @@ -102,7 +107,10 @@ static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type) case CHAR_XDMA_C2H: case CHAR_BYPASS_H2C: case CHAR_BYPASS_C2H: - BUG_ON(!engine); + if (!engine) { + pr_err("Invalid DMA engine\n"); + return rv; + } rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], xdev->idx, engine->channel); break; @@ -133,22 +141,23 @@ int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine) if (!xcdev || xcdev->magic != MAGIC_CHAR) { pr_info("%s, xcdev 0x%p, magic 0x%lx.\n", - fname, xcdev, xcdev ? xcdev->magic : 0xFFFFFFFF); + fname, xcdev, xcdev ? xcdev->magic : 0xFFFFFFFF); return -EINVAL; } - xdev = xcdev->xdev; + xdev = xcdev->xdev; if (!xdev || xdev->magic != MAGIC_DEVICE) { pr_info("%s, xdev 0x%p, magic 0x%lx.\n", - fname, xdev, xdev ? xdev->magic : 0xFFFFFFFF); + fname, xdev, xdev ? xdev->magic : 0xFFFFFFFF); return -EINVAL; } if (check_engine) { - struct xdma_engine *engine = xcdev->engine; + struct xdma_engine *engine = xcdev->engine; + if (!engine || engine->magic != MAGIC_ENGINE) { pr_info("%s, engine 0x%p, magic 0x%lx.\n", fname, - engine, engine ? engine->magic : 0xFFFFFFFF); + engine, engine ? engine->magic : 0xFFFFFFFF); return -EINVAL; } } @@ -162,7 +171,11 @@ int char_open(struct inode *inode, struct file *file) /* pointer to containing structure of the character device inode */ xcdev = container_of(inode->i_cdev, struct xdma_cdev, cdev); - BUG_ON(xcdev->magic != MAGIC_CHAR); + if (xcdev->magic != MAGIC_CHAR) { + pr_err("xcdev 0x%p inode 0x%lx magic mismatch 0x%lx\n", + xcdev, inode->i_ino, xcdev->magic); + return -EINVAL; + } /* create a reference to our char device in the opened file */ file->private_data = xcdev; @@ -177,13 +190,30 @@ int char_close(struct inode *inode, struct file *file) struct xdma_dev *xdev; struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; - BUG_ON(!xcdev); - BUG_ON(xcdev->magic != MAGIC_CHAR); + if (!xcdev) { + pr_err("char device with inode 0x%lx xcdev NULL\n", + inode->i_ino); + return -EINVAL; + } + + if (xcdev->magic != MAGIC_CHAR) { + pr_err("xcdev 0x%p magic mismatch 0x%lx\n", + xcdev, xcdev->magic); + return -EINVAL; + } /* fetch device specific data stored earlier during open */ xdev = xcdev->xdev; - BUG_ON(!xdev); - BUG_ON(xdev->magic != MAGIC_DEVICE); + if (!xdev) { + pr_err("char device with inode 0x%lx xdev NULL\n", + inode->i_ino); + return -EINVAL; + } + + if (xdev->magic != MAGIC_DEVICE) { + pr_err("xdev 0x%p magic mismatch 0x%lx\n", xdev, xdev->magic); + return -EINVAL; + } return 0; } @@ -197,40 +227,52 @@ int char_close(struct inode *inode, struct file *file) static int create_sys_device(struct xdma_cdev *xcdev, enum cdev_type type) { - struct xdma_dev *xdev = xcdev->xdev; - struct xdma_engine *engine = xcdev->engine; - int last_param; + struct xdma_dev *xdev = xcdev->xdev; + struct xdma_engine *engine = xcdev->engine; + int last_param; - if (type == CHAR_EVENTS) - last_param = xcdev->bar; - else - last_param = engine ? engine->channel : 0; + if (type == CHAR_EVENTS) + last_param = xcdev->bar; + else + last_param = engine ? engine->channel : 0; - xcdev->sys_device = device_create(g_xdma_class, &xdev->pdev->dev, - xcdev->cdevno, NULL, devnode_names[type], xdev->idx, - last_param); + xcdev->sys_device = device_create(g_xdma_class, &xdev->pdev->dev, + xcdev->cdevno, NULL, devnode_names[type], xdev->idx, + last_param); - if (!xcdev->sys_device) { - pr_err("device_create(%s) failed\n", devnode_names[type]); - return -1; - } + if (!xcdev->sys_device) { + pr_err("device_create(%s) failed\n", devnode_names[type]); + return -1; + } - return 0; + return 0; } static int destroy_xcdev(struct xdma_cdev *cdev) { if (!cdev) { pr_warn("cdev NULL.\n"); - return 0; + return -EINVAL; } if (cdev->magic != MAGIC_CHAR) { pr_warn("cdev 0x%p magic mismatch 0x%lx\n", cdev, cdev->magic); - return 0; + return -EINVAL; + } + + if (!cdev->xdev) { + pr_err("xdev NULL\n"); + return -EINVAL; + } + + if (!g_xdma_class) { + pr_err("g_xdma_class NULL\n"); + return -EINVAL; + } + + if (!cdev->sys_device) { + pr_err("cdev sys_device NULL\n"); + return -EINVAL; } - BUG_ON(!cdev->xdev); - BUG_ON(!g_xdma_class); - BUG_ON(!cdev->sys_device); if (cdev->sys_device) device_destroy(g_xdma_class, cdev->cdevno); @@ -341,58 +383,91 @@ static int create_xcdev(struct xdma_pci_dev *xpdev, struct xdma_cdev *xcdev, del_cdev: cdev_del(&xcdev->cdev); unregister_region: - unregister_chrdev_region(dev, XDMA_MINOR_COUNT); + unregister_chrdev_region(xcdev->cdevno, XDMA_MINOR_COUNT); return rv; } void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev) { - int i; - + int i = 0; + int rv; #ifdef __XDMA_SYSFS__ - device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance); + device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance); #endif if (xpdev_flag_test(xpdev, XDF_CDEV_SG)) { /* iterate over channels */ - for (i = 0; i < xpdev->h2c_channel_max; i++) + for (i = 0; i < xpdev->h2c_channel_max; i++) { /* remove SG DMA character device */ - destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]); - for (i = 0; i < xpdev->c2h_channel_max; i++) - destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]); + rv = destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]); + if (rv < 0) + pr_err("Failed to destroy h2c xcdev %d error :0x%x\n", + i, rv); + } + for (i = 0; i < xpdev->c2h_channel_max; i++) { + rv = destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]); + if (rv < 0) + pr_err("Failed to destroy c2h xcdev %d error 0x%x\n", + i, rv); + } } if (xpdev_flag_test(xpdev, XDF_CDEV_EVENT)) { - for (i = 0; i < xpdev->user_max; i++) - destroy_xcdev(&xpdev->events_cdev[i]); + for (i = 0; i < xpdev->user_max; i++) { + rv = destroy_xcdev(&xpdev->events_cdev[i]); + if (rv < 0) + pr_err("Failed to destroy cdev event %d error 0x%x\n", + i, rv); + } } /* remove control character device */ if (xpdev_flag_test(xpdev, XDF_CDEV_CTRL)) { - destroy_xcdev(&xpdev->ctrl_cdev); + rv = destroy_xcdev(&xpdev->ctrl_cdev); + if (rv < 0) + pr_err("Failed to destroy cdev ctrl event %d error 0x%x\n", + i, rv); } /* remove user character device */ if (xpdev_flag_test(xpdev, XDF_CDEV_USER)) { - destroy_xcdev(&xpdev->user_cdev); + rv = destroy_xcdev(&xpdev->user_cdev); + if (rv < 0) + pr_err("Failed to destroy user cdev %d error 0x%x\n", + i, rv); } if (xpdev_flag_test(xpdev, XDF_CDEV_XVC)) { - destroy_xcdev(&xpdev->xvc_cdev); + rv = destroy_xcdev(&xpdev->xvc_cdev); + if (rv < 0) + pr_err("Failed to destroy xvc cdev %d error 0x%x\n", + i, rv); } if (xpdev_flag_test(xpdev, XDF_CDEV_BYPASS)) { /* iterate over channels */ - for (i = 0; i < xpdev->h2c_channel_max; i++) + for (i = 0; i < xpdev->h2c_channel_max; i++) { /* remove DMA Bypass character device */ - destroy_xcdev(&xpdev->bypass_h2c_cdev[i]); - for (i = 0; i < xpdev->c2h_channel_max; i++) - destroy_xcdev(&xpdev->bypass_c2h_cdev[i]); - destroy_xcdev(&xpdev->bypass_cdev_base); + rv = destroy_xcdev(&xpdev->bypass_h2c_cdev[i]); + if (rv < 0) + pr_err("Failed to destroy bypass h2c cdev %d error 0x%x\n", + i, rv); + } + for (i = 0; i < xpdev->c2h_channel_max; i++) { + rv = destroy_xcdev(&xpdev->bypass_c2h_cdev[i]); + if (rv < 0) + pr_err("Failed to destroy bypass c2h %d error 0x%x\n", + i, rv); + } + rv = destroy_xcdev(&xpdev->bypass_cdev_base); + if (rv < 0) + pr_err("Failed to destroy base cdev\n"); } if (xpdev->major) - unregister_chrdev_region(MKDEV(xpdev->major, XDMA_MINOR_BASE), XDMA_MINOR_COUNT); + unregister_chrdev_region( + MKDEV(xpdev->major, XDMA_MINOR_BASE), + XDMA_MINOR_COUNT); } int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) @@ -454,7 +529,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) /* ??? Bypass */ /* Initialize Bypass Character Device */ - if (xdev->bypass_bar_idx > 0){ + if (xdev->bypass_bar_idx > 0) { for (i = 0; i < xpdev->h2c_channel_max; i++) { engine = &xdev->engine_h2c[i]; @@ -519,7 +594,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) rv = device_create_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance); if (rv) { - pr_err("Failed to create device file \n"); + pr_err("Failed to create device file\n"); goto fail; } #endif @@ -535,16 +610,33 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) int xdma_cdev_init(void) { g_xdma_class = class_create(THIS_MODULE, XDMA_NODE_NAME); - if (IS_ERR(g_xdma_class)) { - dbg_init(XDMA_NODE_NAME ": failed to create class"); - return -1; - } + if (IS_ERR(g_xdma_class)) { + dbg_init(XDMA_NODE_NAME ": failed to create class"); + return -EINVAL; + } + + /* using kmem_cache_create to enable sequential cleanup */ + cdev_cache = kmem_cache_create("cdev_cache", + sizeof(struct cdev_async_io), 0, + SLAB_HWCACHE_ALIGN, NULL); + + if (!cdev_cache) { + pr_info("memory allocation for cdev_cache failed. OOM\n"); + return -ENOMEM; + } + + xdma_threads_create(num_online_cpus()); return 0; } void xdma_cdev_cleanup(void) { + if (cdev_cache) + kmem_cache_destroy(cdev_cache); + if (g_xdma_class) class_destroy(g_xdma_class); + + xdma_threads_destroy(); } diff --git a/sdk/linux_kernel_drivers/xdma/xdma_cdev.h b/sdk/linux_kernel_drivers/xdma/xdma_cdev.h index 47441fcaf..3361e8ebd 100644 --- a/sdk/linux_kernel_drivers/xdma/xdma_cdev.h +++ b/sdk/linux_kernel_drivers/xdma/xdma_cdev.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef __XDMA_CHRDEV_H__ #define __XDMA_CHRDEV_H__ @@ -39,13 +40,13 @@ int xdma_cdev_init(void); int char_open(struct inode *inode, struct file *file); int char_close(struct inode *inode, struct file *file); -int xcdev_check(const char *, struct xdma_cdev *, bool); - +int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine); void cdev_ctrl_init(struct xdma_cdev *xcdev); void cdev_xvc_init(struct xdma_cdev *xcdev); void cdev_event_init(struct xdma_cdev *xcdev); void cdev_sgdma_init(struct xdma_cdev *xcdev); void cdev_bypass_init(struct xdma_cdev *xcdev); +long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev); int xpdev_create_interfaces(struct xdma_pci_dev *xpdev); diff --git a/sdk/linux_kernel_drivers/xdma/xdma_ioctl.h b/sdk/linux_kernel_drivers/xdma/xdma_ioctl.h deleted file mode 100755 index a250a1de8..000000000 --- a/sdk/linux_kernel_drivers/xdma/xdma_ioctl.h +++ /dev/null @@ -1,78 +0,0 @@ -/******************************************************************************* - * - * Xilinx XDMA IP Core Linux Driver - * - * Copyright(c) Sidebranch. - * Copyright(c) Xilinx, Inc. - * - * Karen Xie - * Leon Woestenberg - * - ******************************************************************************/ -#ifndef _XDMA_IOCALLS_POSIX_H_ -#define _XDMA_IOCALLS_POSIX_H_ - -#include - -/* Use 'x' as magic number */ -#define XDMA_IOC_MAGIC 'x' -/* XL OpenCL X->58(ASCII), L->6C(ASCII), O->0 C->C L->6C(ASCII); */ -#define XDMA_XCL_MAGIC 0X586C0C6C - -#define IOCTL_XDMA_PERF_V1 (1) -#define XDMA_ADDRMODE_MEMORY (0) -#define XDMA_ADDRMODE_FIXED (1) - -/* - * S means "Set" through a ptr, - * T means "Tell" directly with the argument value - * G means "Get": reply by setting through a pointer - * Q means "Query": response is on the return value - * X means "eXchange": switch G and S atomically - * H means "sHift": switch T and Q atomically - * - * _IO(type,nr) no arguments - * _IOR(type,nr,datatype) read data from driver - * _IOW(type,nr.datatype) write data to driver - * _IORW(type,nr,datatype) read/write data - * - * _IOC_DIR(nr) returns direction - * _IOC_TYPE(nr) returns magic - * _IOC_NR(nr) returns number - * _IOC_SIZE(nr) returns size - */ - -enum XDMA_IOC_TYPES { - XDMA_IOC_NOP, - XDMA_IOC_INFO, - XDMA_IOC_MAX -}; - -struct xdma_ioc_base { - unsigned int magic; - unsigned int command; -}; - -struct xdma_ioc_info { - struct xdma_ioc_base base; - unsigned short vendor; - unsigned short device; - unsigned short subsystem_vendor; - unsigned short subsystem_device; - unsigned dma_engine_version; - unsigned driver_version; - unsigned long long feature_id; - unsigned short domain; - unsigned char bus; - unsigned char dev; - unsigned char func; -}; - -/* IOCTL codes */ -#define XDMA_IOCINFO _IOWR(XDMA_IOC_MAGIC, XDMA_IOC_INFO, struct xdma_ioc_info) - -#define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int) -#define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int) -#define IOCTL_XDMA_ALIGN_GET _IOR('q', 6, int) - -#endif /* _XDMA_IOCALLS_POSIX_H_ */ diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.c b/sdk/linux_kernel_drivers/xdma/xdma_mod.c index 3b0943220..2ee9218ce 100755 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2016 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include @@ -35,8 +36,7 @@ #include "version.h" #define DRV_MODULE_NAME "xdma" -#define DRV_MODULE_DESC "Xilinx XDMA Classic Driver" -#define DRV_MODULE_RELDATE "Feb. 2017" +#define DRV_MODULE_DESC "Xilinx XDMA Reference Driver" static char version[] = DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; @@ -47,48 +47,48 @@ MODULE_VERSION(DRV_MODULE_VERSION); MODULE_LICENSE("GPL v2"); /* SECTION: Module global variables */ -static int xpdev_cnt = 0; +static int xpdev_cnt; static const struct pci_device_id pci_ids[] = { { PCI_DEVICE(0x10ee, 0x903f), }, { PCI_DEVICE(0x10ee, 0x9038), }, { PCI_DEVICE(0x10ee, 0x9028), }, - { PCI_DEVICE(0x10ee, 0x9018), }, + { PCI_DEVICE(0x10ee, 0x9018), }, { PCI_DEVICE(0x10ee, 0x9034), }, { PCI_DEVICE(0x10ee, 0x9024), }, - { PCI_DEVICE(0x10ee, 0x9014), }, + { PCI_DEVICE(0x10ee, 0x9014), }, { PCI_DEVICE(0x10ee, 0x9032), }, { PCI_DEVICE(0x10ee, 0x9022), }, - { PCI_DEVICE(0x10ee, 0x9012), }, + { PCI_DEVICE(0x10ee, 0x9012), }, { PCI_DEVICE(0x10ee, 0x9031), }, { PCI_DEVICE(0x10ee, 0x9021), }, - { PCI_DEVICE(0x10ee, 0x9011), }, + { PCI_DEVICE(0x10ee, 0x9011), }, { PCI_DEVICE(0x10ee, 0x8011), }, { PCI_DEVICE(0x10ee, 0x8012), }, - { PCI_DEVICE(0x10ee, 0x8014), }, - { PCI_DEVICE(0x10ee, 0x8018), }, - { PCI_DEVICE(0x10ee, 0x8021), }, - { PCI_DEVICE(0x10ee, 0x8022), }, - { PCI_DEVICE(0x10ee, 0x8024), }, - { PCI_DEVICE(0x10ee, 0x8028), }, - { PCI_DEVICE(0x10ee, 0x8031), }, - { PCI_DEVICE(0x10ee, 0x8032), }, - { PCI_DEVICE(0x10ee, 0x8034), }, - { PCI_DEVICE(0x10ee, 0x8038), }, - - { PCI_DEVICE(0x10ee, 0x7011), }, - { PCI_DEVICE(0x10ee, 0x7012), }, - { PCI_DEVICE(0x10ee, 0x7014), }, - { PCI_DEVICE(0x10ee, 0x7018), }, - { PCI_DEVICE(0x10ee, 0x7021), }, - { PCI_DEVICE(0x10ee, 0x7022), }, - { PCI_DEVICE(0x10ee, 0x7024), }, + { PCI_DEVICE(0x10ee, 0x8014), }, + { PCI_DEVICE(0x10ee, 0x8018), }, + { PCI_DEVICE(0x10ee, 0x8021), }, + { PCI_DEVICE(0x10ee, 0x8022), }, + { PCI_DEVICE(0x10ee, 0x8024), }, + { PCI_DEVICE(0x10ee, 0x8028), }, + { PCI_DEVICE(0x10ee, 0x8031), }, + { PCI_DEVICE(0x10ee, 0x8032), }, + { PCI_DEVICE(0x10ee, 0x8034), }, + { PCI_DEVICE(0x10ee, 0x8038), }, + + { PCI_DEVICE(0x10ee, 0x7011), }, + { PCI_DEVICE(0x10ee, 0x7012), }, + { PCI_DEVICE(0x10ee, 0x7014), }, + { PCI_DEVICE(0x10ee, 0x7018), }, + { PCI_DEVICE(0x10ee, 0x7021), }, + { PCI_DEVICE(0x10ee, 0x7022), }, + { PCI_DEVICE(0x10ee, 0x7024), }, { PCI_DEVICE(0x10ee, 0x7028), }, - { PCI_DEVICE(0x10ee, 0x7031), }, - { PCI_DEVICE(0x10ee, 0x7032), }, - { PCI_DEVICE(0x10ee, 0x7034), }, - { PCI_DEVICE(0x10ee, 0x7038), }, + { PCI_DEVICE(0x10ee, 0x7031), }, + { PCI_DEVICE(0x10ee, 0x7032), }, + { PCI_DEVICE(0x10ee, 0x7034), }, + { PCI_DEVICE(0x10ee, 0x7038), }, { PCI_DEVICE(0x10ee, 0x6828), }, { PCI_DEVICE(0x10ee, 0x6830), }, @@ -105,13 +105,12 @@ static const struct pci_device_id pci_ids[] = { { PCI_DEVICE(0x10ee, 0x4B28), }, { PCI_DEVICE(0x10ee, 0x2808), }, + { PCI_DEVICE(0x10ee, 0xf000), }, + { PCI_DEVICE(0x10ee, 0xf001), }, - { PCI_DEVICE(0x10ee, 0x2808), }, - - { PCI_DEVICE(0x1d0f, 0xf000), }, - { PCI_DEVICE(0x1d0f, 0xf001), }, - { PCI_DEVICE(0x1d0f, 0x1042), }, - +#ifdef INTERNAL_TESTING + { PCI_DEVICE(0x1d0f, 0x1042), 0}, +#endif {0,} }; MODULE_DEVICE_TABLE(pci, pci_ids); @@ -132,7 +131,7 @@ static void xpdev_free(struct xdma_pci_dev *xpdev) static struct xdma_pci_dev *xpdev_alloc(struct pci_dev *pdev) { - struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL); + struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL); if (!xpdev) return NULL; @@ -161,12 +160,28 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max, &xpdev->h2c_channel_max, &xpdev->c2h_channel_max); - if (!hndl) - return -EINVAL; + if (!hndl) { + rv = -EINVAL; + goto err_out; + } - BUG_ON(xpdev->user_max > MAX_USER_IRQ); - BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX); - BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX); + if (xpdev->user_max > MAX_USER_IRQ) { + pr_err("Maximum users limit reached\n"); + rv = -EINVAL; + goto err_out; + } + + if (xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) { + pr_err("Maximun H2C channel limit reached\n"); + rv = -EINVAL; + goto err_out; + } + + if (xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) { + pr_err("Maximun C2H channel limit reached\n"); + rv = -EINVAL; + goto err_out; + } if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max) pr_warn("NO engine found!\n"); @@ -183,9 +198,15 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) xdev = xdev_find_by_pdev(pdev); if (!xdev) { pr_warn("NO xdev found!\n"); - return -EINVAL; + rv = -EINVAL; + goto err_out; + } + + if (hndl != xdev) { + pr_err("xdev handle mismatch\n"); + rv = -EINVAL; + goto err_out; } - BUG_ON(hndl != xdev ); pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n", dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev, @@ -198,11 +219,11 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) if (rv) goto err_out; - dev_set_drvdata(&pdev->dev, xpdev); + dev_set_drvdata(&pdev->dev, xpdev); return 0; -err_out: +err_out: pr_err("pdev 0x%p, err %d.\n", pdev, rv); xpdev_free(xpdev); return rv; @@ -223,7 +244,7 @@ static void remove_one(struct pci_dev *pdev) pdev, xpdev, xpdev->xdev); xpdev_free(xpdev); - dev_set_drvdata(&pdev->dev, NULL); + dev_set_drvdata(&pdev->dev, NULL); } static pci_ers_result_t xdma_error_detected(struct pci_dev *pdev, @@ -274,7 +295,7 @@ static void xdma_error_resume(struct pci_dev *pdev) pci_cleanup_aer_uncorrect_error_status(pdev); } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE static void xdma_reset_prepare(struct pci_dev *pdev) { struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); @@ -291,7 +312,7 @@ static void xdma_reset_done(struct pci_dev *pdev) xdma_device_online(pdev, xpdev->xdev); } -#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE static void xdma_reset_notify(struct pci_dev *pdev, bool prepare) { struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); @@ -309,10 +330,10 @@ static const struct pci_error_handlers xdma_err_handler = { .error_detected = xdma_error_detected, .slot_reset = xdma_slot_reset, .resume = xdma_error_resume, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE .reset_prepare = xdma_reset_prepare, .reset_done = xdma_reset_done, -#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE .reset_notify = xdma_reset_notify, #endif }; @@ -328,8 +349,6 @@ static struct pci_driver pci_driver = { static int __init xdma_mod_init(void) { int rv; - extern unsigned int desc_blen_max; - extern unsigned int sgdma_timeout; pr_info("%s", version); diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.h b/sdk/linux_kernel_drivers/xdma/xdma_mod.h index 0ede7a080..2bf25fe70 100755 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.h +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2017 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,7 @@ * Karen Xie * ******************************************************************************/ + #ifndef __XDMA_MODULE_H__ #define __XDMA_MODULE_H__ @@ -48,14 +49,19 @@ #include #include #include +#include #include "libxdma.h" +#include "xdma_thread.h" #define MAGIC_ENGINE 0xEEEEEEEEUL #define MAGIC_DEVICE 0xDDDDDDDDUL #define MAGIC_CHAR 0xCCCCCCCCUL #define MAGIC_BITSTREAM 0xBBBBBBBBUL +extern unsigned int desc_blen_max; +extern unsigned int sgdma_timeout; + struct xdma_cdev { unsigned long magic; /* structure ID for sanity checks */ struct xdma_pci_dev *xpdev; @@ -98,12 +104,19 @@ struct xdma_pci_dev { void *data; }; -struct xdma_io_cb { - void __user *buf; - size_t len; - unsigned int pages_nr; - struct sg_table sgt; - struct page **pages; +struct cdev_async_io { + struct kiocb *iocb; + struct xdma_io_cb *cb; + bool write; + bool cancel; + int cmpl_cnt; + int req_cnt; + spinlock_t lock; + struct work_struct wrk_itm; + struct cdev_async_io *next; + ssize_t res; + ssize_t res2; + int err_cnt; }; #endif /* ifndef __XDMA_MODULE_H__ */ diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.c b/sdk/linux_kernel_drivers/xdma/xdma_thread.c new file mode 100644 index 000000000..80f1244d8 --- /dev/null +++ b/sdk/linux_kernel_drivers/xdma/xdma_thread.c @@ -0,0 +1,345 @@ +/******************************************************************************* + * + * Xilinx XDMA IP Core Linux Driver + * Copyright(c) 2017 - 2020 Xilinx, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + * + * The full GNU General Public License is included in this distribution in + * the file called "LICENSE". + * + * Karen Xie + * + ******************************************************************************/ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_thread.h" + +#include +#include + + +/* ********************* global variables *********************************** */ +static struct xdma_kthread *cs_threads; +static unsigned int thread_cnt; + + +/* ********************* static function definitions ************************ */ +static int xdma_thread_cmpl_status_pend(struct list_head *work_item) +{ + struct xdma_engine *engine = list_entry(work_item, struct xdma_engine, + cmplthp_list); + int pend = 0; + unsigned long flags; + + spin_lock_irqsave(&engine->lock, flags); + pend = !list_empty(&engine->transfer_list); + spin_unlock_irqrestore(&engine->lock, flags); + + return pend; +} + +static int xdma_thread_cmpl_status_proc(struct list_head *work_item) +{ + struct xdma_engine *engine; + struct xdma_transfer * transfer; + + engine = list_entry(work_item, struct xdma_engine, cmplthp_list); + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + engine_service_poll(engine, transfer->desc_num); + return 0; +} + + + +static inline int xthread_work_pending(struct xdma_kthread *thp) +{ + struct list_head *work_item, *next; + + /* any work items assigned to this thread? */ + if (list_empty(&thp->work_list)) + return 0; + + + /* any work item has pending work to do? */ + list_for_each_safe(work_item, next, &thp->work_list) { + if (thp->fpending && thp->fpending(work_item)) + return 1; + + } + return 0; +} + +static inline void xthread_reschedule(struct xdma_kthread *thp) +{ + if (thp->timeout) { + pr_debug_thread("%s rescheduling for %u seconds", + thp->name, thp->timeout); + wait_event_interruptible_timeout(thp->waitq, thp->schedule, + msecs_to_jiffies(thp->timeout)); + } else { + pr_debug_thread("%s rescheduling", thp->name); + wait_event_interruptible(thp->waitq, thp->schedule); + } +} + +static int xthread_main(void *data) +{ + struct xdma_kthread *thp = (struct xdma_kthread *)data; + + pr_debug_thread("%s UP.\n", thp->name); + + disallow_signal(SIGPIPE); + + if (thp->finit) + thp->finit(thp); + + + while (!kthread_should_stop()) { + + struct list_head *work_item, *next; + + pr_debug_thread("%s interruptible\n", thp->name); + + /* any work to do? */ + lock_thread(thp); + if (!xthread_work_pending(thp)) { + unlock_thread(thp); + xthread_reschedule(thp); + lock_thread(thp); + } + thp->schedule = 0; + + if (thp->work_cnt) { + pr_debug_thread("%s processing %u work items\n", + thp->name, thp->work_cnt); + /* do work */ + list_for_each_safe(work_item, next, &thp->work_list) { + thp->fproc(work_item); + } + } + unlock_thread(thp); + schedule(); + } + + pr_debug_thread("%s, work done.\n", thp->name); + + if (thp->fdone) + thp->fdone(thp); + + pr_debug_thread("%s, exit.\n", thp->name); + return 0; +} + + +int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id) +{ + int len; + int node; + + if (thp->task) { + pr_warn("kthread %s task already running?\n", thp->name); + return -EINVAL; + } + + len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id); + if (len < 0) + return -EINVAL; + + thp->id = id; + + spin_lock_init(&thp->lock); + INIT_LIST_HEAD(&thp->work_list); + init_waitqueue_head(&thp->waitq); + + node = cpu_to_node(thp->cpu); + pr_debug("node : %d\n", node); + + thp->task = kthread_create_on_node(xthread_main, (void *)thp, + node, "%s", thp->name); + if (IS_ERR(thp->task)) { + pr_err("kthread %s, create task failed: 0x%lx\n", + thp->name, (unsigned long)IS_ERR(thp->task)); + thp->task = NULL; + return -EFAULT; + } + + kthread_bind(thp->task, thp->cpu); + + pr_debug_thread("kthread 0x%p, %s, cpu %u, task 0x%p.\n", + thp, thp->name, thp->cpu, thp->task); + + wake_up_process(thp->task); + return 0; +} + + +int xdma_kthread_stop(struct xdma_kthread *thp) +{ + int rv; + + if (!thp->task) { + pr_debug_thread("kthread %s, already stopped.\n", thp->name); + return 0; + } + + thp->schedule = 1; + rv = kthread_stop(thp->task); + if (rv < 0) { + pr_warn("kthread %s, stop err %d.\n", thp->name, rv); + return rv; + } + + pr_debug_thread("kthread %s, 0x%p, stopped.\n", thp->name, thp->task); + thp->task = NULL; + + return 0; +} + + + +void xdma_thread_remove_work(struct xdma_engine *engine) +{ + struct xdma_kthread *cmpl_thread; + unsigned long flags; + + spin_lock_irqsave(&engine->lock, flags); + cmpl_thread = engine->cmplthp; + engine->cmplthp = NULL; + +// pr_debug("%s removing from thread %s, %u.\n", +// descq->conf.name, cmpl_thread ? cmpl_thread->name : "?", +// cpu_idx); + + spin_unlock_irqrestore(&engine->lock, flags); + +#if 0 + if (cpu_idx < cpu_count) { + spin_lock(&qcnt_lock); + per_cpu_qcnt[cpu_idx]--; + spin_unlock(&qcnt_lock); + } +#endif + + if (cmpl_thread) { + lock_thread(cmpl_thread); + list_del(&engine->cmplthp_list); + cmpl_thread->work_cnt--; + unlock_thread(cmpl_thread); + } +} + +void xdma_thread_add_work(struct xdma_engine *engine) +{ + struct xdma_kthread *thp = cs_threads; + unsigned int v = 0; + int i, idx = thread_cnt; + unsigned long flags; + + /* Polled mode only */ + for (i = 0; i < thread_cnt; i++, thp++) { + lock_thread(thp); + if (idx == thread_cnt) { + v = thp->work_cnt; + idx = i; + } else if (!thp->work_cnt) { + idx = i; + unlock_thread(thp); + break; + } else if (thp->work_cnt < v) + idx = i; + unlock_thread(thp); + } + + thp = cs_threads + idx; + lock_thread(thp); + list_add_tail(&engine->cmplthp_list, &thp->work_list); + engine->intr_work_cpu = idx; + thp->work_cnt++; + unlock_thread(thp); + + pr_info("%s 0x%p assigned to cmpl status thread %s,%u.\n", + engine->name, engine, thp->name, thp->work_cnt); + + + spin_lock_irqsave(&engine->lock, flags); + engine->cmplthp = thp; + spin_unlock_irqrestore(&engine->lock, flags); +} + +int xdma_threads_create(unsigned int num_threads) +{ + struct xdma_kthread *thp; + int rv; + int cpu; + + if (thread_cnt) { + pr_warn("threads already created!"); + return 0; + } + + pr_info("xdma_threads_create\n"); + + cs_threads = kzalloc(num_threads * sizeof(struct xdma_kthread), + GFP_KERNEL); + if (!cs_threads) + return -ENOMEM; + + /* N dma writeback monitoring threads */ + thp = cs_threads; + for_each_online_cpu(cpu) { + pr_debug("index %d cpu %d online\n", thread_cnt, cpu); + thp->cpu = cpu; + thp->timeout = 0; + thp->fproc = xdma_thread_cmpl_status_proc; + thp->fpending = xdma_thread_cmpl_status_pend; + rv = xdma_kthread_start(thp, "cmpl_status_th", thread_cnt); + if (rv < 0) + goto cleanup_threads; + + thread_cnt++; + if (thread_cnt == num_threads) + break; + thp++; + } + + return 0; + +cleanup_threads: + kfree(cs_threads); + cs_threads = NULL; + thread_cnt = 0; + + return rv; +} + +void xdma_threads_destroy(void) +{ + int i; + struct xdma_kthread *thp; + + if (!thread_cnt) + return; + + /* N dma writeback monitoring threads */ + thp = cs_threads; + for (i = 0; i < thread_cnt; i++, thp++) + if (thp->fproc) + xdma_kthread_stop(thp); + + kfree(cs_threads); + cs_threads = NULL; + thread_cnt = 0; +} diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.h b/sdk/linux_kernel_drivers/xdma/xdma_thread.h new file mode 100644 index 000000000..919fe9263 --- /dev/null +++ b/sdk/linux_kernel_drivers/xdma/xdma_thread.h @@ -0,0 +1,152 @@ +/******************************************************************************* + * + * Xilinx XDMA IP Core Linux Driver + * Copyright(c) 2017 - 2020 Xilinx, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + * + * The full GNU General Public License is included in this distribution in + * the file called "LICENSE". + * + * Karen Xie + * + ******************************************************************************/ + +#ifndef __XDMA_KTHREAD_H__ +#define __XDMA_KTHREAD_H__ +/** + * @file + * @brief This file contains the declarations for xdma kernel threads + * + */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "libxdma.h" + +#ifdef DEBUG_THREADS +#define lock_thread(thp) \ + do { \ + pr_debug("locking thp %s ...\n", (thp)->name); \ + spin_lock(&(thp)->lock); \ + } while (0) + +#define unlock_thread(thp) \ + do { \ + pr_debug("unlock thp %s ...\n", (thp)->name); \ + spin_unlock(&(thp)->lock); \ + } while (0) + +#define xdma_kthread_wakeup(thp) \ + do { \ + pr_info("signaling thp %s ...\n", (thp)->name); \ + wake_up_process((thp)->task); \ + } while (0) + +#define pr_debug_thread(fmt, ...) pr_info(fmt, __VA_ARGS__) + +#else +/** lock thread macro */ +#define lock_thread(thp) spin_lock(&(thp)->lock) +/** un lock thread macro */ +#define unlock_thread(thp) spin_unlock(&(thp)->lock) +#define xdma_kthread_wakeup(thp) \ + do { \ + thp->schedule = 1; \ + wake_up_interruptible(&thp->waitq); \ + } while (0) +/** pr_debug_thread */ +#define pr_debug_thread(fmt, ...) +#endif + +/** + * @struct - xdma_kthread + * @brief xdma thread book keeping parameters + */ +struct xdma_kthread { + /** thread lock*/ + spinlock_t lock; + /** name of the thread */ + char name[16]; + /** cpu number for which the thread associated with */ + unsigned short cpu; + /** thread id */ + unsigned short id; + /** thread sleep timeout value */ + unsigned int timeout; + /** flags for thread */ + unsigned long flag; + /** thread wait queue */ + wait_queue_head_t waitq; + /* flag to indicate scheduling of thread */ + unsigned int schedule; + /** kernel task structure associated with thread*/ + struct task_struct *task; + /** thread work list count */ + unsigned int work_cnt; + /** thread work list count */ + struct list_head work_list; + /** thread initialization handler */ + int (*finit)(struct xdma_kthread *); + /** thread pending handler */ + int (*fpending)(struct list_head *); + /** thread peocessing handler */ + int (*fproc)(struct list_head *); + /** thread done handler */ + int (*fdone)(struct xdma_kthread *); +}; + + +/*****************************************************************************/ +/** + * xdma_threads_create() - create xdma threads +*********/ +int xdma_threads_create(unsigned int num_threads); + +/*****************************************************************************/ +/** + * xdma_threads_destroy() - destroy all the xdma threads created + * during system initialization + * + * @return none + *****************************************************************************/ +void xdma_threads_destroy(void); + +/*****************************************************************************/ +/** + * xdma_thread_remove_work() - handler to remove the attached work thread + * + * @param[in] engine: pointer to xdma_engine + * + * @return none + *****************************************************************************/ +void xdma_thread_remove_work(struct xdma_engine *engine); + +/*****************************************************************************/ +/** + * xdma_thread_add_work() - handler to add a work thread + * + * @param[in] engine: pointer to xdma_engine + * + * @return none + *****************************************************************************/ +void xdma_thread_add_work(struct xdma_engine *engine); + +#endif /* #ifndef __XDMA_KTHREAD_H__ */ From 3aaec77a8f57436c7eeef5f1a5cf18834fff12a8 Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Mon, 24 Aug 2020 15:42:43 -0700 Subject: [PATCH 2/5] xdma: add back F1 specific device ids --- sdk/linux_kernel_drivers/xdma/xdma_mod.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.c b/sdk/linux_kernel_drivers/xdma/xdma_mod.c index 2ee9218ce..193029304 100755 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.c @@ -105,8 +105,8 @@ static const struct pci_device_id pci_ids[] = { { PCI_DEVICE(0x10ee, 0x4B28), }, { PCI_DEVICE(0x10ee, 0x2808), }, - { PCI_DEVICE(0x10ee, 0xf000), }, - { PCI_DEVICE(0x10ee, 0xf001), }, + { PCI_DEVICE(0x1d0f, 0xf000), }, + { PCI_DEVICE(0x1d0f, 0xf001), }, #ifdef INTERNAL_TESTING { PCI_DEVICE(0x1d0f, 0x1042), 0}, From c23868d944f65dd61cfd7eae2d1df05ad8279cc1 Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Mon, 31 Aug 2020 17:12:07 -0700 Subject: [PATCH 3/5] xdma: update license header check --- sdk/linux_kernel_drivers/xdma/xdma_mod.c | 2 +- sdk/linux_kernel_drivers/xdma/xdma_thread.c | 2 +- sdk/linux_kernel_drivers/xdma/xdma_thread.h | 2 +- shared/lib/check_src_headers.py | 9 +-------- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.c b/sdk/linux_kernel_drivers/xdma/xdma_mod.c index 193029304..c9672069e 100755 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2016 - 2020 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.c b/sdk/linux_kernel_drivers/xdma/xdma_thread.c index 80f1244d8..8b57f14be 100644 --- a/sdk/linux_kernel_drivers/xdma/xdma_thread.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_thread.c @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2017 - 2020 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.h b/sdk/linux_kernel_drivers/xdma/xdma_thread.h index 919fe9263..bb5c15fd5 100644 --- a/sdk/linux_kernel_drivers/xdma/xdma_thread.h +++ b/sdk/linux_kernel_drivers/xdma/xdma_thread.h @@ -1,7 +1,7 @@ /******************************************************************************* * * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2017 - 2020 Xilinx, Inc. + * Copyright(c) 2015 - 2020 Xilinx, Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, diff --git a/shared/lib/check_src_headers.py b/shared/lib/check_src_headers.py index 8d390a3cd..fca1e8204 100755 --- a/shared/lib/check_src_headers.py +++ b/shared/lib/check_src_headers.py @@ -176,7 +176,7 @@ ''' xilinx_xdma1 = '''Xilinx XDMA IP Core Linux Driver -Copyright(c) 2015 - 2017 Xilinx, Inc. +Copyright(c) 2015 - 2020 Xilinx, Inc. This program is free software; you can redistribute it and/or modify it under the terms and conditions of the GNU General Public License, @@ -194,12 +194,6 @@ the file called "LICENSE". ''' -xilinx_xdma2 = '''Xilinx XDMA IP Core Linux Driver - -Copyright(c) Sidebranch. -Copyright(c) Xilinx, Inc. -''' - xilinx1 = '''\xa9 Copyright 2017 Xilinx, Inc. All rights reserved. This file contains confidential and proprietary information of Xilinx, Inc. and is protected under U.S. and @@ -400,7 +394,6 @@ apache_header_2018.split("\n"), gpl2_header.split("\n"), xilinx_xdma1.split("\n"), - xilinx_xdma2.split("\n"), xilinx1.split("\n"), xilinx2_header.split("\n"), xilinx3_header.split("\n"), From e26887ca60b9717f147f70072f7b00aaf5facab1 Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Tue, 22 Sep 2020 17:59:33 -0700 Subject: [PATCH 4/5] keep enable_credit_mp disabled --- sdk/linux_kernel_drivers/xdma/cdev_xvc.c | 2 +- sdk/linux_kernel_drivers/xdma/libxdma.c | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sdk/linux_kernel_drivers/xdma/cdev_xvc.c b/sdk/linux_kernel_drivers/xdma/cdev_xvc.c index b2ea63484..e346bc79f 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_xvc.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_xvc.c @@ -219,7 +219,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) unlock: #if KERNEL_VERSION(5, 1, 0) >= LINUX_VERSION_CODE - mmiowb(); + wmb(); #endif spin_unlock(&xcdev->lock); diff --git a/sdk/linux_kernel_drivers/xdma/libxdma.c b/sdk/linux_kernel_drivers/xdma/libxdma.c index d91bcf724..5728a600d 100755 --- a/sdk/linux_kernel_drivers/xdma/libxdma.c +++ b/sdk/linux_kernel_drivers/xdma/libxdma.c @@ -64,11 +64,10 @@ static unsigned int interrupt_mode; module_param(interrupt_mode, uint, 0644); MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); -static unsigned int enable_credit_mp = 1; +static unsigned int enable_credit_mp = 0; module_param(enable_credit_mp, uint, 0644); -MODULE_PARM_DESC( - enable_credit_mp, - "Set 0 to disable credit feature, default is 1 ( credit control enabled)"); +MODULE_PARM_DESC(enable_credit_mp, + "Set 1 to enable credit feature, default is 0 (no credit control)"); unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; module_param(desc_blen_max, uint, 0644); From 801f36a2c20f8e9d5924178b2f29e3840b0c581d Mon Sep 17 00:00:00 2001 From: Karen Xie Date: Sun, 4 Oct 2020 20:42:01 -0700 Subject: [PATCH 5/5] back out experimental aio code --- sdk/linux_kernel_drivers/xdma/10-xdma.rules | 0 sdk/linux_kernel_drivers/xdma/Makefile | 3 +- sdk/linux_kernel_drivers/xdma/cdev_bypass.c | 2 + sdk/linux_kernel_drivers/xdma/cdev_sgdma.c | 351 +-- sdk/linux_kernel_drivers/xdma/cdev_xvc.h | 3 +- sdk/linux_kernel_drivers/xdma/libxdma.c | 2586 +++++++------------ sdk/linux_kernel_drivers/xdma/libxdma.h | 84 +- sdk/linux_kernel_drivers/xdma/libxdma_api.h | 20 - sdk/linux_kernel_drivers/xdma/version.h | 2 +- sdk/linux_kernel_drivers/xdma/xdma_cdev.c | 18 - sdk/linux_kernel_drivers/xdma/xdma_mod.c | 4 + sdk/linux_kernel_drivers/xdma/xdma_mod.h | 20 +- sdk/linux_kernel_drivers/xdma/xdma_thread.c | 345 --- sdk/linux_kernel_drivers/xdma/xdma_thread.h | 152 -- 14 files changed, 949 insertions(+), 2641 deletions(-) mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/10-xdma.rules mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/Makefile mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/libxdma.c mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/libxdma.h mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/version.h mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/xdma_mod.c mode change 100755 => 100644 sdk/linux_kernel_drivers/xdma/xdma_mod.h delete mode 100644 sdk/linux_kernel_drivers/xdma/xdma_thread.c delete mode 100644 sdk/linux_kernel_drivers/xdma/xdma_thread.h diff --git a/sdk/linux_kernel_drivers/xdma/10-xdma.rules b/sdk/linux_kernel_drivers/xdma/10-xdma.rules old mode 100755 new mode 100644 diff --git a/sdk/linux_kernel_drivers/xdma/Makefile b/sdk/linux_kernel_drivers/xdma/Makefile old mode 100755 new mode 100644 index b3ad79d2f..2427268a7 --- a/sdk/linux_kernel_drivers/xdma/Makefile +++ b/sdk/linux_kernel_drivers/xdma/Makefile @@ -32,7 +32,7 @@ EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS) #EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__ ifneq ($(KERNELRELEASE),) - $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o + $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o obj-m := $(TARGET_MODULE).o else BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build @@ -42,6 +42,7 @@ all : clean: $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean + @/bin/rm -f *.ko modules.order *.mod.c *.o *.o.ur-safe .*.o.cmd install: all $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules_install diff --git a/sdk/linux_kernel_drivers/xdma/cdev_bypass.c b/sdk/linux_kernel_drivers/xdma/cdev_bypass.c index d38d3fac3..5e40526be 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_bypass.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_bypass.c @@ -21,6 +21,8 @@ * Karen Xie * ******************************************************************************/ +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + #include "libxdma_api.h" #include "xdma_cdev.h" diff --git a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c index 9b344356f..2b615bb15 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c +++ b/sdk/linux_kernel_drivers/xdma/cdev_sgdma.c @@ -21,113 +21,19 @@ * Karen Xie * ******************************************************************************/ - #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include #include -#include -#include -#include -#include -#include -#include -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) -#include -#endif #include "libxdma_api.h" #include "xdma_cdev.h" #include "cdev_sgdma.h" -#include "xdma_thread.h" /* Module Parameters */ unsigned int sgdma_timeout = 10; module_param(sgdma_timeout, uint, 0644); MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec."); - -extern struct kmem_cache *cdev_cache; -static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write); - - -static void async_io_handler(unsigned long cb_hndl, int err) -{ - struct xdma_cdev *xcdev; - struct xdma_engine *engine; - struct xdma_dev *xdev; - struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; - struct cdev_async_io *caio = (struct cdev_async_io *)cb->private; - ssize_t numbytes = 0; - ssize_t res, res2; - int lock_stat; - int rv; - - if (caio == NULL) { - pr_err("Invalid work struct\n"); - return; - } - - xcdev = (struct xdma_cdev *)caio->iocb->ki_filp->private_data; - rv = xcdev_check(__func__, xcdev, 1); - if (rv < 0) - return; - - /* Safeguarding for cancel requests */ - lock_stat = spin_trylock(&caio->lock); - if (!lock_stat) { - pr_err("caio lock not acquired\n"); - goto skip_dev_lock; - } - - if (false != caio->cancel) { - pr_err("skipping aio\n"); - goto skip_tran; - } - - engine = xcdev->engine; - xdev = xcdev->xdev; - - if (!err) - numbytes = xdma_xfer_completion((void *)cb, xdev, - engine->channel, cb->write, cb->ep_addr, - &cb->sgt, 0, sgdma_timeout * 1000); - - char_sgdma_unmap_user_buf(cb, cb->write); - - caio->res2 |= (err < 0) ? err : 0; - if (caio->res2) - caio->err_cnt++; - - caio->cmpl_cnt++; - caio->res += numbytes; - - if (caio->cmpl_cnt == caio->req_cnt) { - res = caio->res; - res2 = caio->res2; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) - caio->iocb->ki_complete(caio->iocb, res, res2); -#else - aio_complete(caio->iocb, res, res2); -#endif -skip_tran: - spin_unlock(&caio->lock); - kmem_cache_free(cdev_cache, caio); - kfree(cb); - return; - } - spin_unlock(&caio->lock); - return; - -skip_dev_lock: -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) - caio->iocb->ki_complete(caio->iocb, numbytes, -EBUSY); -#else - aio_complete(caio->iocb, numbytes, -EBUSY); -#endif - kmem_cache_free(cdev_cache, caio); -} - - /* * character device file operations for SG DMA engine */ @@ -151,12 +57,9 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence) if (newpos < 0) return -EINVAL; file->f_pos = newpos; - dbg_fops("%s: pos=%lld\n", __func__, (signed long long)newpos); -#if 0 - pr_err("0x%p, off %lld, whence %d -> pos %lld.\n", + dbg_fops("0x%p, off %lld, whence %d -> pos %lld.\n", file, (signed long long)off, whence, (signed long long)off); -#endif return newpos; } @@ -273,7 +176,7 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) { struct sg_table *sgt = &cb->sgt; unsigned long len = cb->len; - void __user *buf = cb->buf; + char __user *buf = cb->buf; struct scatterlist *sg; unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE - 1) - ((unsigned long)buf & PAGE_MASK)) @@ -326,8 +229,8 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) sg = sgt->sgl; for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) { unsigned int offset = offset_in_page(buf); - unsigned int nbytes = - min_t(unsigned int, PAGE_SIZE - offset, len); + unsigned int nbytes = min_t(unsigned int, + PAGE_SIZE - offset, len); flush_dcache_page(cb->pages[i]); sg_set_page(sg, cb->pages[i], nbytes, offset); @@ -340,6 +243,7 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) pr_err("Invalid user buffer length. Cannot map to sgl\n"); return -EINVAL; } + cb->pages_nr = pages_nr; return 0; @@ -385,15 +289,12 @@ static ssize_t char_sgdma_read_write(struct file *file, const char __user *buf, memset(&cb, 0, sizeof(struct xdma_io_cb)); cb.buf = (char __user *)buf; cb.len = count; - cb.ep_addr = (u64)*pos; - cb.write = write; rv = char_sgdma_map_user_buf_to_sgl(&cb, write); if (rv < 0) return rv; res = xdma_xfer_submit(xdev, engine->channel, write, *pos, &cb.sgt, - 0, sgdma_timeout * 1000); - + 0, sgdma_timeout * 1000); char_sgdma_unmap_user_buf(&cb, write); return res; @@ -401,208 +302,44 @@ static ssize_t char_sgdma_read_write(struct file *file, const char __user *buf, static ssize_t char_sgdma_write(struct file *file, const char __user *buf, - size_t count, loff_t *pos) + size_t count, loff_t *pos) { - return char_sgdma_read_write(file, buf, count, pos, 1); + return char_sgdma_read_write(file, (char *)buf, count, pos, 1); } static ssize_t char_sgdma_read(struct file *file, char __user *buf, size_t count, loff_t *pos) { - return char_sgdma_read_write(file, buf, count, pos, 0); -} - -static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io, - unsigned long count, loff_t pos) -{ - struct xdma_cdev *xcdev = (struct xdma_cdev *) - iocb->ki_filp->private_data; - struct cdev_async_io *caio; - struct xdma_engine *engine; - struct xdma_dev *xdev; - int rv; - unsigned long i; - - if (!xcdev) { - pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n", - iocb->ki_filp, (u64)count, (u64)pos, 1); - return -EINVAL; - } - - engine = xcdev->engine; - xdev = xcdev->xdev; - - if (engine->dir != DMA_TO_DEVICE) { - pr_err("r/w mismatch. WRITE, dir %d.\n", - engine->dir); - return -EINVAL; - } - - caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL); - memset(caio, 0, sizeof(struct cdev_async_io)); - - caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL); - - spin_lock_init(&caio->lock); - iocb->private = caio; - caio->iocb = iocb; - caio->write = true; - caio->cancel = false; - caio->req_cnt = count; - - for (i = 0; i < count; i++) { - - memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb)); - - caio->cb[i].buf = io[i].iov_base; - caio->cb[i].len = io[i].iov_len; - caio->cb[i].ep_addr = (u64)pos; - caio->cb[i].write = true; - caio->cb[i].private = caio; - caio->cb[i].io_done = async_io_handler; - rv = check_transfer_align(engine, caio->cb[i].buf, - caio->cb[i].len, pos, 1); - if (rv) { - pr_info("Invalid transfer alignment detected\n"); - kmem_cache_free(cdev_cache, caio); - return rv; - } - - rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true); - if (rv < 0) - return rv; - - rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, - engine->channel, caio->cb[i].write, - caio->cb[i].ep_addr, &caio->cb[i].sgt, - 0, sgdma_timeout * 1000); - } - - if (engine->cmplthp) - xdma_kthread_wakeup(engine->cmplthp); - - return -EIOCBQUEUED; -} - - -static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io, - unsigned long count, loff_t pos) -{ - - struct xdma_cdev *xcdev = (struct xdma_cdev *) - iocb->ki_filp->private_data; - struct cdev_async_io *caio; - struct xdma_engine *engine; - struct xdma_dev *xdev; - int rv; - unsigned long i; - - if (!xcdev) { - pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n", - iocb->ki_filp, (u64)count, (u64)pos, 1); - return -EINVAL; - } - - engine = xcdev->engine; - xdev = xcdev->xdev; - - if (engine->dir != DMA_FROM_DEVICE) { - pr_err("r/w mismatch. READ, dir %d.\n", - engine->dir); - return -EINVAL; - } - - caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL); - memset(caio, 0, sizeof(struct cdev_async_io)); - - caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL); - - spin_lock_init(&caio->lock); - iocb->private = caio; - caio->iocb = iocb; - caio->write = false; - caio->cancel = false; - caio->req_cnt = count; - - for (i = 0; i < count; i++) { - - memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb)); - - caio->cb[i].buf = io[i].iov_base; - caio->cb[i].len = io[i].iov_len; - caio->cb[i].ep_addr = (u64)pos; - caio->cb[i].write = false; - caio->cb[i].private = caio; - caio->cb[i].io_done = async_io_handler; - - rv = check_transfer_align(engine, caio->cb[i].buf, - caio->cb[i].len, pos, 1); - if (rv) { - pr_info("Invalid transfer alignment detected\n"); - kmem_cache_free(cdev_cache, caio); - return rv; - } - - rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true); - if (rv < 0) - return rv; - - rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, - engine->channel, caio->cb[i].write, - caio->cb[i].ep_addr, &caio->cb[i].sgt, - 0, sgdma_timeout * 1000); - } - - if (engine->cmplthp) - xdma_kthread_wakeup(engine->cmplthp); - - return -EIOCBQUEUED; -} - -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) -static ssize_t cdev_write_iter(struct kiocb *iocb, struct iov_iter *io) -{ - return cdev_aio_write(iocb, io->iov, io->nr_segs, io->iov_offset); + return char_sgdma_read_write(file, (char *)buf, count, pos, 0); } -static ssize_t cdev_read_iter(struct kiocb *iocb, struct iov_iter *io) -{ - return cdev_aio_read(iocb, io->iov, io->nr_segs, io->iov_offset); -} -#endif - static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg) { int rv; struct xdma_dev *xdev; - if (!engine) { - pr_err("Invalid DMA engine\n"); + if (!engine || !engine->xdev) { + pr_err("Invalid DMA engine 0x%p, 0x%p.\n", + engine, engine ? engine->xdev : NULL); return -EINVAL; } xdev = engine->xdev; - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } - /* performance measurement already running on this engine? */ + /* if performance measurement already running on this engine */ if (engine->xdma_perf) { - dbg_perf("IOCTL_XDMA_PERF_START failed!\n"); dbg_perf("Perf measurement already seems to be running!\n"); return -EBUSY; } - engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), - GFP_KERNEL); + engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), + GFP_KERNEL); if (!engine->xdma_perf) return -ENOMEM; rv = copy_from_user(engine->xdma_perf, - (struct xdma_performance_ioctl __user *)arg, - sizeof(struct xdma_performance_ioctl)); - + (struct xdma_performance_ioctl *)arg, + sizeof(struct xdma_performance_ioctl)); if (rv < 0) { dbg_perf("Failed to copy from user space 0x%lx\n", arg); return -EINVAL; @@ -615,16 +352,15 @@ static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg) enable_perf(engine); dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size); + /* initialize wait queue */ -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - init_swait_queue_head(&engine->xdma_perf_wq); -#else init_waitqueue_head(&engine->xdma_perf_wq); -#endif + rv = xdma_performance_submit(xdev, engine); if (rv < 0) pr_err("Failed to submit dma performance\n"); - return rv; + + return 0; } static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg) @@ -633,37 +369,35 @@ static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg) int rv; if (!engine) { - pr_err("Invalid DMA engine\n"); + pr_err("DMA engine NULL.\n"); return -EINVAL; } dbg_perf("IOCTL_XDMA_PERF_STOP\n"); - /* no performance measurement running on this engine? */ + /* if no performance measurement running on this engine */ if (!engine->xdma_perf) { dbg_perf("No measurement in progress\n"); return -EINVAL; } /* stop measurement */ + dbg_perf("Waiting for measurement to stop\n"); transfer = engine_cyclic_stop(engine); if (!transfer) { pr_err("Failed to stop cyclic transfer\n"); return -EINVAL; - } - dbg_perf("Waiting for measurement to stop\n"); + } get_perf_stats(engine); - rv = copy_to_user((void __user *)arg, engine->xdma_perf, sizeof(struct xdma_performance_ioctl)); if (rv) { dbg_perf("Error copying result to user\n"); - return rv; + return -EINVAL; } kfree(transfer); - kfree(engine->xdma_perf); engine->xdma_perf = NULL; @@ -675,20 +409,20 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg) int rc; if (!engine) { - pr_err("Invalid DMA engine\n"); + pr_err("DMA engine NULL.\n"); return -EINVAL; } dbg_perf("IOCTL_XDMA_PERF_GET\n"); if (engine->xdma_perf) { - get_perf_stats(engine); + get_perf_stats(engine); rc = copy_to_user((void __user *)arg, engine->xdma_perf, - sizeof(struct xdma_performance_ioctl)); + sizeof(struct xdma_performance_ioctl)); if (rc) { dbg_perf("Error copying result to user\n"); - return rc; + return -EINVAL; } } else { dbg_perf("engine->xdma_perf == NULL?\n"); @@ -698,18 +432,18 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg) return 0; } -static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) +static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) { return engine_addrmode_set(engine, arg); } -static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) +static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) { int rv; unsigned long src; if (!engine) { - pr_err("Invalid DMA engine\n"); + pr_err("DMA engine NULL.\n"); return -EINVAL; } src = !!engine->non_incr_addr; @@ -723,7 +457,7 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) { if (!engine) { - pr_err("Invalid DMA engine\n"); + pr_err("DMA engine NULL.\n"); return -EINVAL; } @@ -732,12 +466,11 @@ static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) } static long char_sgdma_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) + unsigned long arg) { struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; struct xdma_dev *xdev; struct xdma_engine *engine; - int rv = 0; rv = xcdev_check(__func__, xcdev, 1); @@ -747,7 +480,7 @@ static long char_sgdma_ioctl(struct file *file, unsigned int cmd, xdev = xcdev->xdev; engine = xcdev->engine; - switch (cmd) { + switch (cmd) { case IOCTL_XDMA_PERF_START: rv = ioctl_do_perf_start(engine, arg); break; @@ -767,7 +500,7 @@ static long char_sgdma_ioctl(struct file *file, unsigned int cmd, rv = ioctl_do_align_get(engine, arg); break; default: - dbg_perf("Unsupported operation\n"); + dbg_perf("Unsupported operation 0x%x.\n", cmd); rv = -EINVAL; break; } @@ -819,17 +552,7 @@ static const struct file_operations sgdma_fops = { .open = char_sgdma_open, .release = char_sgdma_close, .write = char_sgdma_write, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) - .write_iter = cdev_write_iter, -#else - .aio_write = cdev_aio_write, -#endif .read = char_sgdma_read, -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) - .read_iter = cdev_read_iter, -#else - .aio_read = cdev_aio_read, -#endif .unlocked_ioctl = char_sgdma_ioctl, .llseek = char_sgdma_llseek, }; diff --git a/sdk/linux_kernel_drivers/xdma/cdev_xvc.h b/sdk/linux_kernel_drivers/xdma/cdev_xvc.h index 8e0a4bec1..9a2b8689f 100644 --- a/sdk/linux_kernel_drivers/xdma/cdev_xvc.h +++ b/sdk/linux_kernel_drivers/xdma/cdev_xvc.h @@ -28,8 +28,7 @@ #include /* - * !!! TODO !!! - * need a better way set the bar offset dynamicly + * the bar offset can be changed at compile time via xvc_bar_offset */ #define XVC_BAR_OFFSET_DFLT 0x40000 /* DSA 4.0 */ diff --git a/sdk/linux_kernel_drivers/xdma/libxdma.c b/sdk/linux_kernel_drivers/xdma/libxdma.c old mode 100755 new mode 100644 index 5728a600d..9dc519b69 --- a/sdk/linux_kernel_drivers/xdma/libxdma.c +++ b/sdk/linux_kernel_drivers/xdma/libxdma.c @@ -22,7 +22,7 @@ * ******************************************************************************/ -#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include #include @@ -35,25 +35,6 @@ #include "libxdma.h" #include "libxdma_api.h" #include "cdev_sgdma.h" -#include "xdma_thread.h" - -/* SECTION: Module licensing */ - -#ifdef __LIBXDMA_MOD__ -#include "version.h" -#define DRV_MODULE_NAME "libxdma" -#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" - -static char version[] = - DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; - -MODULE_AUTHOR("Xilinx, Inc."); -MODULE_DESCRIPTION(DRV_MODULE_DESC); -MODULE_VERSION(DRV_MODULE_VERSION); -MODULE_LICENSE("GPL v2"); -#endif - -extern unsigned int desc_blen_max; /* Module Parameters */ static unsigned int poll_mode; @@ -64,37 +45,13 @@ static unsigned int interrupt_mode; module_param(interrupt_mode, uint, 0644); MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); -static unsigned int enable_credit_mp = 0; +static unsigned int enable_credit_mp; module_param(enable_credit_mp, uint, 0644); -MODULE_PARM_DESC(enable_credit_mp, - "Set 1 to enable credit feature, default is 0 (no credit control)"); +MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; module_param(desc_blen_max, uint, 0644); -MODULE_PARM_DESC(desc_blen_max, - "per descriptor max. buffer length, default is (1 << 28) - 1"); - -#define XDMA_PERF_NUM_DESC 128 - -/* Kernel version adaptative code */ -#if KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE -/* since 4.18, using simple wait queues is not recommended - * except for realtime constraint (see swait.h comments) - * and will likely be removed in future kernel versions - */ -#define xlx_wake_up swake_up_one -#define xlx_wait_event_interruptible_timeout \ - swait_event_interruptible_timeout_exclusive -#elif KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE -#define xlx_wake_up swake_up -#define xlx_wait_event_interruptible_timeout \ - swait_event_interruptible_timeout -#else -#define xlx_wake_up wake_up_interruptible -#define xlx_wait_event_interruptible_timeout \ - wait_event_interruptible_timeout -#endif - +MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); /* * xdma device management @@ -107,7 +64,8 @@ static LIST_HEAD(xdev_rcu_list); static DEFINE_SPINLOCK(xdev_rcu_lock); #ifndef list_last_entry -#define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) #endif static inline void xdev_list_add(struct xdma_dev *xdev) @@ -125,7 +83,7 @@ static inline void xdev_list_add(struct xdma_dev *xdev) mutex_unlock(&xdev_mutex); dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", - dev_name(&xdev->pdev->dev), xdev, xdev->idx); + dev_name(&xdev->pdev->dev), xdev, xdev->idx); spin_lock(&xdev_rcu_lock); list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); @@ -160,10 +118,9 @@ struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) mutex_unlock(&xdev_mutex); return NULL; } -EXPORT_SYMBOL_GPL(xdev_find_by_pdev); static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, - void *hndl) + void *hndl) { struct xdma_dev *xdev; @@ -172,13 +129,13 @@ static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, xdev = xdev_find_by_pdev(pdev); if (!xdev) { - pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", fname, - pdev, hndl); + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", + fname, pdev, hndl); return -EINVAL; } if (xdev != hndl) { - pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", fname, pdev, hndl, - xdev); + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", + fname, pdev, hndl, xdev); return -EINVAL; } @@ -187,8 +144,7 @@ static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, #ifdef __LIBXDMA_DEBUG__ /* SECTION: Function definitions */ -inline void __write_register(const char *fn, u32 value, void *iomem, - unsigned long off) +inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) { pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); iowrite32(value, iomem); @@ -215,9 +171,8 @@ static inline u64 build_u64(u64 hi, u64 lo) static void check_nonzero_interrupt_status(struct xdma_dev *xdev) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); u32 w; w = read_register(®->user_int_enable); @@ -252,9 +207,8 @@ static void check_nonzero_interrupt_status(struct xdma_dev *xdev) /* channel_interrupts_enable -- Enable interrupts we are interested in */ static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); } @@ -262,9 +216,8 @@ static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) /* channel_interrupts_disable -- Disable interrupts we not interested in */ static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); } @@ -272,9 +225,8 @@ static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) /* user_interrupts_enable -- Enable interrupts we are interested in */ static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); } @@ -282,9 +234,8 @@ static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) /* user_interrupts_disable -- Disable interrupts we not interested in */ static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); } @@ -292,19 +243,18 @@ static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) /* read_interrupts -- Print the interrupt controller status */ static u32 read_interrupts(struct xdma_dev *xdev) { - struct interrupt_regs *reg = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); u32 lo; u32 hi; /* extra debugging; inspect complete engine set of registers */ hi = read_register(®->user_int_request); dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", - ®->user_int_request, hi); + ®->user_int_request, hi); lo = read_register(®->channel_int_request); dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", - ®->channel_int_request, lo); + ®->channel_int_request, lo); /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ return build_u32(hi, lo); @@ -316,26 +266,26 @@ void enable_perf(struct xdma_engine *engine) w = XDMA_PERF_CLEAR; write_register(w, &engine->regs->perf_ctrl, - (unsigned long)(&engine->regs->perf_ctrl) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); read_register(&engine->regs->identifier); w = XDMA_PERF_AUTO | XDMA_PERF_RUN; write_register(w, &engine->regs->perf_ctrl, - (unsigned long)(&engine->regs->perf_ctrl) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); read_register(&engine->regs->identifier); dbg_perf("IOCTL_XDMA_PERF_START\n"); + } -EXPORT_SYMBOL_GPL(enable_perf); void get_perf_stats(struct xdma_engine *engine) { u32 hi; u32 lo; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return; } @@ -361,39 +311,38 @@ void get_perf_stats(struct xdma_engine *engine) lo = read_register(&engine->regs->perf_pnd_lo); engine->xdma_perf->pending_count = build_u64(hi, lo); } -EXPORT_SYMBOL_GPL(get_perf_stats); -static int engine_reg_dump(struct xdma_engine *engine) +static void engine_reg_dump(struct xdma_engine *engine) { u32 w; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } w = read_register(&engine->regs->identifier); - pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", engine->name, - &engine->regs->identifier, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", + engine->name, &engine->regs->identifier, w); w &= BLOCK_ID_MASK; if (w != BLOCK_ID_HEAD) { - pr_err("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", - engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); - return -EINVAL; + pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return; } /* extra debugging; inspect complete engine set of registers */ w = read_register(&engine->regs->status); - pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", engine->name, - &engine->regs->status, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", + engine->name, &engine->regs->status, w); w = read_register(&engine->regs->control); - pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", engine->name, - &engine->regs->control, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", + engine->name, &engine->regs->control, w); w = read_register(&engine->sgdma_regs->first_desc_lo); - pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", engine->name, - &engine->sgdma_regs->first_desc_lo, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", + engine->name, &engine->sgdma_regs->first_desc_lo, w); w = read_register(&engine->sgdma_regs->first_desc_hi); - pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", engine->name, - &engine->sgdma_regs->first_desc_hi, w); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", + engine->name, &engine->sgdma_regs->first_desc_hi, w); w = read_register(&engine->sgdma_regs->first_desc_adjacent); pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", engine->name, &engine->sgdma_regs->first_desc_adjacent, w); @@ -403,10 +352,15 @@ static int engine_reg_dump(struct xdma_engine *engine) w = read_register(&engine->regs->interrupt_enable_mask); pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", engine->name, &engine->regs->interrupt_enable_mask, w); - - return 0; } +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return -1 on failure, status register otherwise + */ static void engine_status_dump(struct xdma_engine *engine) { u32 v = engine->status; @@ -415,7 +369,7 @@ static void engine_status_dump(struct xdma_engine *engine) int len = 0; len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); - + if ((v & XDMA_STAT_BUSY)) len += sprintf(buf + len, "BUSY,"); if ((v & XDMA_STAT_DESC_STOPPED)) @@ -461,7 +415,7 @@ static void engine_status_dump(struct xdma_engine *engine) len += sprintf(buf + len, "SLAVE_ERR "); buf[len - 1] = ','; } - + } else { /* C2H only */ if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { @@ -494,55 +448,39 @@ static void engine_status_dump(struct xdma_engine *engine) pr_info("%s\n", buffer); } -/** - * engine_status_read() - read status of SG DMA engine (optionally reset) - * - * Stores status in engine->status. - * - * @return error value on failure, 0 otherwise - */ -static int engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +static void engine_status_read(struct xdma_engine *engine, bool clr, bool dump) { - int rv = 0; - - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } - if (dump) { - rv = engine_reg_dump(engine); - if (rv < 0) { - pr_err("Failed to dump register\n"); - return rv; - } - } + if (dump) + engine_reg_dump(engine); /* read status register */ - if (clear) + if (clr) engine->status = read_register(&engine->regs->status_rc); else engine->status = read_register(&engine->regs->status); if (dump) engine_status_dump(engine); - - return rv; } /** * xdma_engine_stop() - stop an SG DMA engine * */ -static int xdma_engine_stop(struct xdma_engine *engine) +static void xdma_engine_stop(struct xdma_engine *engine) { u32 w; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } - dbg_tfr("%s(engine=%p)\n", __func__, engine); + dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); w = 0; w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; @@ -551,29 +489,33 @@ static int xdma_engine_stop(struct xdma_engine *engine) w |= (u32)XDMA_CTRL_IE_DESC_ERROR; if (poll_mode) { - w |= (u32)XDMA_CTRL_POLL_MODE_WB; + w |= (u32) XDMA_CTRL_POLL_MODE_WB; } else { w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + /* Disable IDLE STOPPED for MM */ + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; } dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", - engine->name, w, (u32 *)&engine->regs->control); + engine->name, w, (u32 *)&engine->regs->control); write_register(w, &engine->regs->control, (unsigned long)(&engine->regs->control) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); /* dummy read of status register to flush all previous writes */ - dbg_tfr("%s(%s) done\n", __func__, engine->name); - return 0; + dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); } -static int engine_start_mode_config(struct xdma_engine *engine) +static void engine_start_mode_config(struct xdma_engine *engine) { u32 w; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } /* If a perf test is running, enable the engine interrupts */ @@ -586,10 +528,9 @@ static int engine_start_mode_config(struct xdma_engine *engine) w |= XDMA_CTRL_IE_READ_ERROR; w |= XDMA_CTRL_IE_DESC_ERROR; - write_register( - w, &engine->regs->interrupt_enable_mask, + write_register(w, &engine->regs->interrupt_enable_mask, (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); } /* write control register of SG DMA engine */ @@ -601,28 +542,29 @@ static int engine_start_mode_config(struct xdma_engine *engine) if (poll_mode) { w |= (u32)XDMA_CTRL_POLL_MODE_WB; - } else { + } else { w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; - } + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + } /* set non-incremental addressing mode */ if (engine->non_incr_addr) w |= (u32)XDMA_CTRL_NON_INCR_ADDR; dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, - (void *)&engine->regs->control); - + (void *)&engine->regs->control); /* start the engine */ write_register(w, &engine->regs->control, - (unsigned long)(&engine->regs->control) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); /* dummy read of status register to flush all previous writes */ w = read_register(&engine->regs->status); dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", - &engine->regs->status, w); - return 0; + &engine->regs->status, w); } /** @@ -646,46 +588,29 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) struct xdma_transfer *transfer; u32 w; int extra_adj = 0; - int rv; - - if (!engine) { - pr_err("dma engine NULL\n"); - return NULL; - } /* engine must be idle */ - if (engine->running) { - pr_info("%s engine is not in idle state to start\n", - engine->name); + if (unlikely(!engine || engine->running)) { + pr_err("engine 0x%p running.\n", engine); return NULL; } - /* engine transfer queue must not be empty */ - if (list_empty(&engine->transfer_list)) { - pr_debug("%s engine transfer queue must not be empty\n", - engine->name); + if (unlikely(list_empty(&engine->transfer_list))) { + pr_err("engine %s queue empty.\n", engine->name); return NULL; } /* inspect first transfer queued on the engine */ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); - if (!transfer) { - pr_debug("%s queued transfer must not be empty\n", - engine->name); + entry); + if (unlikely(!transfer)) { + pr_err("engine %s no xfer queued.\n", engine->name); return NULL; } /* engine is no longer shutdown */ engine->shutdown = ENGINE_SHUTDOWN_NONE; - dbg_tfr("%s(%s): transfer=0x%p.\n", __func__, engine->name, transfer); - - /* Add credits for Streaming mode C2H */ - if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { - if (enable_credit_mp) - write_register(engine->desc_used, - &engine->sgdma_regs->credits, 0); - } + dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); /* initialize number of descriptors of dequeued transfers */ engine->desc_dequeued = 0; @@ -693,48 +618,38 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) /* write lower 32-bit of bus address of transfer first descriptor */ w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, - (void *)&engine->sgdma_regs->first_desc_lo); + (void *)&engine->sgdma_regs->first_desc_lo); write_register(w, &engine->sgdma_regs->first_desc_lo, - (unsigned long)(&engine->sgdma_regs->first_desc_lo) - - (unsigned long)(&engine->sgdma_regs)); + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); /* write upper 32-bit of bus address of transfer first descriptor */ w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, - (void *)&engine->sgdma_regs->first_desc_hi); + (void *)&engine->sgdma_regs->first_desc_hi); write_register(w, &engine->sgdma_regs->first_desc_hi, - (unsigned long)(&engine->sgdma_regs->first_desc_hi) - - (unsigned long)(&engine->sgdma_regs)); + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); if (transfer->desc_adjacent > 0) { extra_adj = transfer->desc_adjacent - 1; if (extra_adj > MAX_EXTRA_ADJ) extra_adj = MAX_EXTRA_ADJ; } - dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", extra_adj, - (void *)&engine->sgdma_regs->first_desc_adjacent); - write_register( - extra_adj, &engine->sgdma_regs->first_desc_adjacent, - (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", + extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - (unsigned long)(&engine->sgdma_regs)); dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", &engine->regs->status); - -#if KERNEL_VERSION(5, 1, 0) >= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE <= KERNEL_VERSION(5, 1, 0) mmiowb(); #endif + engine_start_mode_config(engine); - rv = engine_start_mode_config(engine); - if (rv < 0) { - pr_err("Failed to start engine mode config\n"); - return NULL; - } + engine_status_read(engine, 0, 0); - rv = engine_status_read(engine, 0, 0); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - return NULL; - } dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); /* remember the engine is running */ engine->running = 1; @@ -749,29 +664,22 @@ static struct xdma_transfer *engine_start(struct xdma_engine *engine) * @engine pointer to struct xdma_engine * */ -static int engine_service_shutdown(struct xdma_engine *engine) +static void engine_service_shutdown(struct xdma_engine *engine) { - int rv; /* if the engine stopped with RUN still asserted, de-assert RUN now */ dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); - rv = xdma_engine_stop(engine); - if (rv < 0) { - pr_err("Failed to stop engine\n"); - return rv; - } + xdma_engine_stop(engine); engine->running = 0; /* awake task on engine's shutdown wait queue */ - xlx_wake_up(&engine->shutdown_wq); - return 0; + wake_up_interruptible(&engine->shutdown_wq); } -static struct xdma_transfer *engine_transfer_completion( - struct xdma_engine *engine, +struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, struct xdma_transfer *transfer) { - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return NULL; } @@ -782,33 +690,17 @@ static struct xdma_transfer *engine_transfer_completion( /* synchronous I/O? */ /* awake task on transfer's wait queue */ - xlx_wake_up(&transfer->wq); - - /* Send completion notification for Last transfer */ - if (transfer->cb && transfer->last_in_request) - transfer->cb->io_done((unsigned long)transfer->cb, 0); + wake_up_interruptible(&transfer->wq); return transfer; } -static struct xdma_transfer * -engine_service_transfer_list(struct xdma_engine *engine, - struct xdma_transfer *transfer, - u32 *pdesc_completed) +struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) { - if (!engine) { - pr_err("dma engine NULL\n"); - return NULL; - } - - if (!pdesc_completed) { - pr_err("%s completed descriptors are null.\n", engine->name); - return NULL; - } - - if (unlikely(!transfer)) { - pr_info("%s xfer empty, pdesc completed %u.\n", engine->name, - *pdesc_completed); + if (unlikely(!engine || !pdesc_completed || !transfer)) { + pr_err("engine 0x%p, pdesc_completed 0x%p, xfer 0x%p.\n", + engine, pdesc_completed, transfer); return NULL; } @@ -817,12 +709,11 @@ engine_service_transfer_list(struct xdma_engine *engine, * except for the last (i.e. use > instead of >=). */ while (transfer && (!transfer->cyclic) && - (*pdesc_completed > transfer->desc_num)) { + (*pdesc_completed > transfer->desc_num)) { /* remove this transfer from pdesc_completed */ *pdesc_completed -= transfer->desc_num; dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", engine->name, transfer, transfer->desc_num); - /* remove completed transfer from list */ list_del(engine->transfer_list.next); /* add to dequeued number of descriptors during this run */ @@ -830,16 +721,14 @@ engine_service_transfer_list(struct xdma_engine *engine, /* mark transfer as succesfully completed */ transfer->state = TRANSFER_STATE_COMPLETED; - /* - * Complete transfer - sets transfer to NULL if an async - * transfer has completed - */ + /* Complete transfer - sets transfer to NULL if an async + * transfer has completed */ transfer = engine_transfer_completion(engine, transfer); /* if exists, get the next transfer on the list */ if (!list_empty(&engine->transfer_list)) { transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); + struct xdma_transfer, entry); dbg_tfr("Non-completed transfer %p\n", transfer); } else { /* no further transfers? */ @@ -850,11 +739,11 @@ engine_service_transfer_list(struct xdma_engine *engine, return transfer; } -static int engine_err_handle(struct xdma_engine *engine, - struct xdma_transfer *transfer, u32 desc_completed) +static void engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) { u32 value; - int rv = 0; + /* * The BUSY bit is expected to be clear now but older HW has a race * condition which could cause it to be still set. If it's set, re-read @@ -863,49 +752,36 @@ static int engine_err_handle(struct xdma_engine *engine, if (engine->status & XDMA_STAT_BUSY) { value = read_register(&engine->regs->status); if ((value & XDMA_STAT_BUSY)) - printk_ratelimited(KERN_INFO "%s has errors but is still BUSY\n", - engine->name); + printk_ratelimited(KERN_INFO + "%s has errors but is still BUSY\n", + engine->name); } - printk_ratelimited(KERN_INFO "%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + printk_ratelimited(KERN_INFO + "%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", engine->name, engine->status, transfer, desc_completed, transfer->desc_num); - + /* mark transfer as failed */ transfer->state = TRANSFER_STATE_FAILED; - rv = xdma_engine_stop(engine); - if (rv < 0) - pr_err("Failed to stop engine\n"); - return rv; + xdma_engine_stop(engine); } -static struct xdma_transfer * -engine_service_final_transfer(struct xdma_engine *engine, - struct xdma_transfer *transfer, - u32 *pdesc_completed) +struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) { - if (!engine) { - pr_err("dma engine NULL\n"); - return NULL; - } - - if (!pdesc_completed) { - pr_err("%s completed descriptors are null.\n", engine->name); + if (unlikely(!engine || !pdesc_completed || !transfer)) { + pr_err("engine 0x%p, pdesc_completed 0x%p, xfer 0x%p.\n", + engine, pdesc_completed, transfer); return NULL; } - /* inspect the current transfer */ - if (unlikely(!transfer)) { - pr_info("%s xfer empty, pdesc completed %u.\n", engine->name, - *pdesc_completed); - return NULL; - } if (((engine->dir == DMA_FROM_DEVICE) && (engine->status & XDMA_STAT_C2H_ERR_MASK)) || ((engine->dir == DMA_TO_DEVICE) && (engine->status & XDMA_STAT_H2C_ERR_MASK))) { - pr_info("engine %s, status error 0x%x.\n", engine->name, - engine->status); + pr_info("engine %s, status error 0x%x.\n", + engine->name, engine->status); engine_status_dump(engine); engine_err_handle(engine, transfer, *pdesc_completed); goto transfer_del; @@ -913,7 +789,7 @@ engine_service_final_transfer(struct xdma_engine *engine, if (engine->status & XDMA_STAT_BUSY) pr_debug("engine %s is unexpectedly busy - ignoring\n", - engine->name); + engine->name); /* the engine stopped on current transfer? */ if (*pdesc_completed < transfer->desc_num) { @@ -934,7 +810,7 @@ engine_service_final_transfer(struct xdma_engine *engine, */ WARN_ON(*pdesc_completed > transfer->desc_num); } - /* mark transfer as successfully completed */ + /* mark transfer as succesfully completed */ transfer->state = TRANSFER_STATE_COMPLETED; } @@ -953,11 +829,11 @@ engine_service_final_transfer(struct xdma_engine *engine, return transfer; } -static int engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) { - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } /* performance measurement is running? */ @@ -966,7 +842,7 @@ static int engine_service_perf(struct xdma_engine *engine, u32 desc_completed) if (engine->status & XDMA_STAT_DESC_COMPLETED) { engine->xdma_perf->iterations = desc_completed; dbg_perf("transfer->xdma_perf->iterations=%d\n", - engine->xdma_perf->iterations); + engine->xdma_perf->iterations); } /* a descriptor stopped the engine? */ @@ -976,35 +852,33 @@ static int engine_service_perf(struct xdma_engine *engine, u32 desc_completed) * wake any XDMA_PERF_IOCTL_STOP waiting for * the performance run to finish */ - xlx_wake_up(&engine->xdma_perf_wq); + wake_up_interruptible(&engine->xdma_perf_wq); dbg_perf("transfer->xdma_perf stopped\n"); } } - return 0; } -static int engine_transfer_dequeue(struct xdma_engine *engine) +static void engine_transfer_dequeue(struct xdma_engine *engine) { struct xdma_transfer *transfer; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } /* pick first transfer on the queue (was submitted to the engine) */ transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); - if (!transfer || transfer != &engine->cyclic_req->tfer[1]) { - pr_err("%s, xfer 0x%p != 0x%p.\n", engine->name, transfer, - &engine->cyclic_req->tfer[1]); - return -EINVAL; + entry); + if (!transfer || transfer != &engine->cyclic_req->xfer) { + pr_info("%s, xfer 0x%p != 0x%p.\n", + engine->name, transfer, &engine->cyclic_req->xfer); + return; } dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", engine->name, transfer, transfer->desc_num); /* remove completed transfer from list */ list_del(engine->transfer_list.next); - return 0; } static int engine_ring_process(struct xdma_engine *engine) @@ -1013,35 +887,31 @@ static int engine_ring_process(struct xdma_engine *engine) int start; int eop_count = 0; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || !engine->cyclic_result)) { + pr_err("engine 0x%p, cyclic_result 0x%p.\n", + engine, engine ? engine->cyclic_result : NULL); return -EINVAL; } - result = engine->cyclic_result; - if (!result) { - pr_err("%s Cyclic transfer resources not available.\n", - engine->name); - return -EINVAL; - } /* where we start receiving in the ring buffer */ start = engine->rx_tail; /* iterate through all newly received RX result descriptors */ - dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", engine->name, - engine->rx_tail, result[engine->rx_tail].status, + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_tail, result[engine->rx_tail].status, result[engine->rx_tail].length); while (result[engine->rx_tail].status && !engine->rx_overrun) { /* EOP bit set in result? */ - if (result[engine->rx_tail].status & RX_STATUS_EOP) + if (result[engine->rx_tail].status & RX_STATUS_EOP){ eop_count++; + } /* increment tail pointer */ engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; - dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", engine->name, - engine->rx_head, engine->rx_tail, + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_head, engine->rx_tail, result[engine->rx_tail].status, result[engine->rx_tail].length); @@ -1058,23 +928,17 @@ static int engine_ring_process(struct xdma_engine *engine) static int engine_service_cyclic_polled(struct xdma_engine *engine) { - int eop_count; + int eop_count = 0; int rc = 0; struct xdma_poll_wb *writeback_data; u32 sched_limit = 0; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); - return -EINVAL; - } - - eop_count = engine->eop_count; writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; while (eop_count == 0) { @@ -1092,36 +956,23 @@ static int engine_service_cyclic_polled(struct xdma_engine *engine) eop_count = engine_ring_process(engine); if (eop_count < 0) { - pr_err("Failed to process engine ring\n"); + pr_err("%s failed to process engine ring\n", + engine->name); return eop_count; } } if (eop_count == 0) { - rc = engine_status_read(engine, 1, 0); - if (rc < 0) { - pr_err("Failed to read engine status\n"); - return rc; - } + engine_status_read(engine, 1, 0); if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { /* transfers on queue? */ - if (!list_empty(&engine->transfer_list)) { - rc = engine_transfer_dequeue(engine); - if (rc < 0) { - pr_err("Failed to dequeue transfer\n"); - return rc; - } - } + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); - rc = engine_service_shutdown(engine); - if (rc < 0) { - pr_err("Failed to shutdown engine\n"); - return rc; - } + engine_service_shutdown(engine); } } - eop_count--; - engine->eop_count = eop_count; + return rc; } @@ -1129,62 +980,40 @@ static int engine_service_cyclic_interrupt(struct xdma_engine *engine) { int eop_count = 0; struct xdma_transfer *xfer; - int rv; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); - return -EINVAL; - } + engine_status_read(engine, 1, 0); - rv = engine_status_read(engine, 1, 0); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - return rv; - } eop_count = engine_ring_process(engine); - if (eop_count < 0) { - pr_err("Failed to process engine ring\n"); - return eop_count; - } /* * wake any reader on EOP, as one or more packets are now in * the RX buffer */ - xfer = &engine->cyclic_req->tfer[0]; - if (enable_credit_mp) { - xlx_wake_up(&xfer->wq); - } else { + xfer = &engine->cyclic_req->xfer; + if(enable_credit_mp){ + wake_up_interruptible(&xfer->wq); + }else{ if (eop_count > 0) { /* awake task on transfer's wait queue */ dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); engine->eop_found = 1; - xlx_wake_up(&xfer->wq); + wake_up_interruptible(&xfer->wq); } } /* engine was running but is no longer busy? */ if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { /* transfers on queue? */ - if (!list_empty(&engine->transfer_list)) { - rv = engine_transfer_dequeue(engine); - if (rv < 0) { - pr_err("Failed to dequeue transfer\n"); - return rv; - } - } + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); - rv = engine_service_shutdown(engine); - if (rv < 0) { - pr_err("Failed to shutdown engine\n"); - return rv; - } + engine_service_shutdown(engine); } return 0; @@ -1195,16 +1024,11 @@ static int engine_service_cyclic(struct xdma_engine *engine) { int rc = 0; - dbg_tfr("%s()\n", __func__); - - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } + dbg_tfr("engine_service_cyclic()"); - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } @@ -1216,13 +1040,14 @@ static int engine_service_cyclic(struct xdma_engine *engine) return rc; } -static int engine_service_resume(struct xdma_engine *engine) + +static void engine_service_resume(struct xdma_engine *engine) { struct xdma_transfer *transfer_started; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } /* engine stopped? */ @@ -1232,16 +1057,18 @@ static int engine_service_resume(struct xdma_engine *engine) /* (re)start engine */ transfer_started = engine_start(engine); if (!transfer_started) { - pr_err("Failed to start dma engine\n"); - return -EINVAL; + pr_err("%s failed to start dma engine\n", + engine->name); + return; } dbg_tfr("re-started %s engine with pending xfer 0x%p\n", engine->name, transfer_started); - /* engine was requested to be shutdown? */ + + /* engine was requested to be shutdown? */ } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { engine->shutdown |= ENGINE_SHUTDOWN_IDLE; /* awake task on engine's shutdown wait queue */ - xlx_wake_up(&engine->shutdown_wq); + wake_up_interruptible(&engine->shutdown_wq); } else { dbg_tfr("no pending transfers, %s engine stays idle.\n", engine->name); @@ -1254,7 +1081,6 @@ static int engine_service_resume(struct xdma_engine *engine) WARN_ON(1); } } - return 0; } /** @@ -1273,8 +1099,8 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) int rv = 0; struct xdma_poll_wb *wb_data; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return -EINVAL; } @@ -1285,11 +1111,7 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) /* Service the engine */ if (!engine->running) { dbg_tfr("Engine was not running!!! Clearing status\n"); - rv = engine_status_read(engine, 1, 0); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - return rv; - } + engine_status_read(engine, 1, 0); return 0; } @@ -1298,26 +1120,17 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) * engine status. For polled mode descriptor completion, this read is * unnecessary and is skipped to reduce latency */ - if ((desc_count == 0) || (err_flag != 0)) { - rv = engine_status_read(engine, 1, 0); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - return rv; - } - } + if ((desc_count == 0) || (err_flag != 0)) + engine_status_read(engine, 1, 0); /* * engine was running but is no longer busy, or writeback occurred, * shut down */ if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || - (desc_count != 0)) { - rv = engine_service_shutdown(engine); - if (rv < 0) { - pr_err("Failed to shutdown engine\n"); - return rv; - } - } + (desc_count != 0)) + engine_service_shutdown(engine); + /* * If called from the ISR, or if an error occurred, the descriptor * count will be zero. In this scenario, read the descriptor count @@ -1332,7 +1145,7 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) if (!list_empty(&engine->transfer_list)) { /* pick first transfer on queue (was submitted to the engine) */ transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); + struct xdma_transfer, entry); dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", transfer, (int)transfer->desc_num); @@ -1341,11 +1154,7 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) (int)desc_count, (int)desc_count - engine->desc_dequeued); - rv = engine_service_perf(engine, desc_count); - if (rv < 0) { - pr_err("Failed to service descriptors\n"); - return rv; - } + engine_service_perf(engine, desc_count); } /* account for already dequeued transfers during this engine run */ @@ -1367,11 +1176,9 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback) } /* Restart the engine following the servicing */ - rv = engine_service_resume(engine); - if (rv < 0) - pr_err("Failed to resume engine\n"); + engine_service_resume(engine); - return rv; + return 0; } /* engine_service_work */ @@ -1379,68 +1186,57 @@ static void engine_service_work(struct work_struct *work) { struct xdma_engine *engine; unsigned long flags; - int rv; engine = container_of(work, struct xdma_engine, work); - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return; } /* lock the engine */ spin_lock_irqsave(&engine->lock, flags); - dbg_tfr("engine_service() for %s engine %p\n", engine->name, engine); - if (engine->cyclic_req) { - rv = engine_service_cyclic(engine); - if (rv < 0) { - pr_err("Failed to service cyclic engine\n"); - goto unlock; - } - } else { - rv = engine_service(engine, 0); - if (rv < 0) { - pr_err("Failed to service engine\n"); - goto unlock; - } - } + dbg_tfr("engine_service() for %s engine %p\n", + engine->name, engine); + if (engine->cyclic_req) + engine_service_cyclic(engine); + else + engine_service(engine, 0); + /* re-enable interrupts for this engine */ - if (engine->xdev->msix_enabled) { - write_register( - engine->interrupt_enable_mask_value, - &engine->regs->interrupt_enable_mask_w1s, - (unsigned long)(&engine->regs - ->interrupt_enable_mask_w1s) - - (unsigned long)(&engine->regs)); + if (engine->xdev->msix_enabled){ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); } else channel_interrupts_enable(engine->xdev, engine->irq_bitmask); - + /* unlock the engine */ -unlock: spin_unlock_irqrestore(&engine->lock, flags); } -static u32 engine_service_wb_monitor(struct xdma_engine *engine, - u32 expected_wb) +static int engine_service_wb_monitor(struct xdma_engine *engine, + u32 expected_wb, u32 *wb) { struct xdma_poll_wb *wb_data; u32 desc_wb = 0; u32 sched_limit = 0; unsigned long timeout; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return -EINVAL; } wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; /* - * Poll the writeback location for the expected number of - * descriptors / error events This loop is skipped for cyclic mode, - * where the expected_desc_count passed in is zero, since it cannot be - * determined before the function is called - */ + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); while (expected_wb != 0) { @@ -1448,9 +1244,10 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, if (desc_wb & WB_ERR_MASK) break; - else if (desc_wb >= expected_wb) + else if (desc_wb == expected_wb) break; - + + /* RTO - prevent system from hanging in polled mode */ if (time_after(jiffies, timeout)) { dbg_tfr("Polling timeout occurred"); dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, @@ -1462,9 +1259,9 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, } /* - * Define NUM_POLLS_PER_SCHED to limit how much time is spent - * in the scheduler - */ + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ if (sched_limit != 0) { if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) @@ -1473,25 +1270,21 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine, sched_limit++; } - return desc_wb; + *wb = desc_wb; + return 0; } -int engine_service_poll(struct xdma_engine *engine, - u32 expected_desc_count) +static int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) { struct xdma_poll_wb *writeback_data; u32 desc_wb = 0; unsigned long flags; int rv = 0; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } @@ -1503,13 +1296,15 @@ int engine_service_poll(struct xdma_engine *engine, } /* - * Poll the writeback location for the expected number of - * descriptors / error events This loop is skipped for cyclic mode, - * where the expected_desc_count passed in is zero, since it cannot be - * determined before the function is called - */ + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ - desc_wb = engine_service_wb_monitor(engine, expected_desc_count); + rv = engine_service_wb_monitor(engine, expected_desc_count, &desc_wb); + if (rv < 0) + return rv; spin_lock_irqsave(&engine->lock, flags); dbg_tfr("%s service.\n", engine->name); @@ -1526,8 +1321,8 @@ static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) { unsigned long flags; - if (!user_irq) { - pr_err("Invalid user_irq\n"); + if (unlikely(!user_irq)) { + pr_err("user_irq NULL.\n"); return IRQ_NONE; } @@ -1557,21 +1352,16 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) struct xdma_dev *xdev; struct interrupt_regs *irq_regs; - dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); - if (!dev_id) { - pr_err("Invalid dev_id on irq line %d\n", irq); - return -IRQ_NONE; - } - xdev = (struct xdma_dev *)dev_id; - - if (!xdev) { - WARN_ON(!xdev); - dbg_irq("%s(irq=%d) xdev=%p ??\n", __func__, irq, xdev); + if (unlikely(!dev_id)) { + pr_err("irq %d, xdev NULL.\n", irq); return IRQ_NONE; } + dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); + xdev = (struct xdma_dev *)dev_id; + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + XDMA_OFS_INT_CTRL); /* read channel interrupt requests */ ch_irq = read_register(&irq_regs->channel_int_request); @@ -1590,8 +1380,8 @@ static irqreturn_t xdma_isr(int irq, void *dev_id) if (user_irq) { int user = 0; - u32 mask = 1; - int max = xdev->user_max; + u32 mask = 1; + int max = xdev->h2c_channel_max; for (; user < max && user_irq; user++, mask <<= 1) { if (user_irq & mask) { @@ -1652,12 +1442,12 @@ static irqreturn_t xdma_user_irq(int irq, void *dev_id) { struct xdma_user_irq *user_irq; - dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); - - if (!dev_id) { - pr_err("Invalid dev_id on irq line %d\n", irq); + if (unlikely(!dev_id)) { + pr_err("irq %d, dev_id NULL.\n", irq); return IRQ_NONE; } + + dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); user_irq = (struct xdma_user_irq *)dev_id; return user_irq_service(irq, user_irq); @@ -1674,29 +1464,29 @@ static irqreturn_t xdma_channel_irq(int irq, void *dev_id) struct xdma_engine *engine; struct interrupt_regs *irq_regs; - dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); - if (!dev_id) { - pr_err("Invalid dev_id on irq line %d\n", irq); + if (unlikely(!dev_id)) { + pr_err("irq %d, dev_id NULL.\n", irq); return IRQ_NONE; } + dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); engine = (struct xdma_engine *)dev_id; xdev = engine->xdev; - if (!xdev) { - WARN_ON(!xdev); - dbg_irq("%s(irq=%d) xdev=%p ??\n", __func__, irq, xdev); + if (unlikely(!xdev)) { + pr_err("xdma_channel_irq(irq=%d) engine 0x%p, xdev NULL.\n", + irq, engine); return IRQ_NONE; } irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + XDMA_OFS_INT_CTRL); /* Disable the interrupt for this engine */ - write_register( - engine->interrupt_enable_mask_value, - &engine->regs->interrupt_enable_mask_w1c, - (unsigned long)(&engine->regs->interrupt_enable_mask_w1c) - + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long) + (&engine->regs->interrupt_enable_mask_w1c) - (unsigned long)(&engine->regs)); /* Dummy read to flush the above write */ read_register(&irq_regs->channel_int_pending); @@ -1774,20 +1564,20 @@ static int is_config_bar(struct xdma_dev *xdev, int idx) int flag = 0; u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ struct interrupt_regs *irq_regs = - (struct interrupt_regs *)(xdev->bar[idx] + XDMA_OFS_INT_CTRL); + (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); struct config_regs *cfg_regs = (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); irq_id = read_register(&irq_regs->identifier); cfg_id = read_register(&cfg_regs->identifier); - if (((irq_id & mask) == IRQ_BLOCK_ID) && - ((cfg_id & mask) == CONFIG_BLOCK_ID)) { + if (((irq_id & mask)== IRQ_BLOCK_ID) && + ((cfg_id & mask)== CONFIG_BLOCK_ID)) { dbg_init("BAR %d is the XDMA config BAR\n", idx); flag = 1; } else { dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", - idx, irq_id, cfg_id); + idx, irq_id, cfg_id); flag = 0; } @@ -1795,8 +1585,8 @@ static int is_config_bar(struct xdma_dev *xdev, int idx) } #ifndef XDMA_CONFIG_BAR_NUM -static int identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, - int config_bar_pos) +static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) { /* * The following logic identifies which BARs contain what functionality @@ -1812,18 +1602,13 @@ static int identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, * correctly with both 32-bit and 64-bit BARs. */ - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } - - if (!bar_id_list) { - pr_err("Invalid bar id list.\n"); - return -EINVAL; + if (unlikely(!xdev || !bar_id_list)) { + pr_err("xdev 0x%p, bar_id_list 0x%p.\n", xdev, bar_id_list); + return; } - dbg_init("xdev 0x%p, bars %d, config at %d.\n", xdev, num_bars, - config_bar_pos); + dbg_init("xdev 0x%p, bars %d, config at %d.\n", + xdev, num_bars, config_bar_pos); switch (num_bars) { case 1: @@ -1859,10 +1644,11 @@ static int identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", num_bars); break; + } - pr_info("%d BARs: config %d, user %d, bypass %d.\n", num_bars, - config_bar_pos, xdev->user_bar_idx, xdev->bypass_bar_idx); - return 0; + pr_info("%d BARs: config %d, user %d, bypass %d.\n", + num_bars, config_bar_pos, xdev->user_bar_idx, + xdev->bypass_bar_idx); } #endif @@ -1911,6 +1697,7 @@ static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) /* Try to identify BAR as XDMA control BAR */ if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { + if (is_config_bar(xdev, i)) { xdev->config_bar_idx = i; config_bar_pos = bar_id_idx; @@ -1930,11 +1717,7 @@ static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) goto fail; } - rv = identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); - if (rv < 0) { - pr_err("Failed to identify bars\n"); - return rv; - } + identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); /* successfully mapped all required BAR regions */ return 0; @@ -2001,24 +1784,19 @@ static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) { int rv = 0; - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } - - if (!pdev) { - pr_err("Invalid pdev\n"); + if (unlikely(!xdev || !pdev)) { + pr_err("xdev 0x%p, pdev 0x%p.\n", xdev, pdev); return -EINVAL; } if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + - xdev->user_max; + xdev->user_max; -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) dbg_init("Enabling MSI-X\n"); rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, - PCI_IRQ_MSIX); + PCI_IRQ_MSIX); #else int i; @@ -2056,7 +1834,7 @@ static void pci_check_intr_pend(struct pci_dev *pdev) pci_read_config_word(pdev, PCI_STATUS, &v); if (v & PCI_STATUS_INTERRUPT) { pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", - dev_name(&pdev->dev), v); + dev_name(&pdev->dev), v); pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); } } @@ -2082,9 +1860,9 @@ static void pci_keep_intx_enabled(struct pci_dev *pdev) static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) { /* user */ - struct interrupt_regs *int_regs = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; u32 max = i + xdev->user_max; int j; @@ -2099,22 +1877,21 @@ static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) else for (k = 0; k < 4 && i < max; i++, k++, shift += 8) val |= (i & 0x1f) << shift; - - write_register( - val, &int_regs->user_msi_vector[j], + + write_register(val, &int_regs->user_msi_vector[j], XDMA_OFS_INT_CTRL + - ((unsigned long)&int_regs->user_msi_vector[j] - - (unsigned long)int_regs)); + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); dbg_init("vector %d, 0x%x.\n", j, val); } } -static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) { - struct interrupt_regs *int_regs = - (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + - XDMA_OFS_INT_CTRL); + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; u32 i; int j; @@ -2130,12 +1907,11 @@ static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) else for (k = 0; k < 4 && i < max; i++, k++, shift += 8) val |= (i & 0x1f) << shift; - + write_register(val, &int_regs->channel_msi_vector[j], - XDMA_OFS_INT_CTRL + - ((unsigned long)&int_regs - ->channel_msi_vector[j] - - (unsigned long)int_regs)); + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->channel_msi_vector[j] - + (unsigned long)int_regs)); dbg_init("vector %d, 0x%x.\n", j, val); } } @@ -2156,7 +1932,7 @@ static void irq_msix_channel_teardown(struct xdma_dev *xdev) if (!engine->msix_irq_line) break; dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, - engine); + engine); free_irq(engine->msix_irq_line, engine); } @@ -2165,7 +1941,7 @@ static void irq_msix_channel_teardown(struct xdma_dev *xdev) if (!engine->msix_irq_line) break; dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, - engine); + engine); free_irq(engine->msix_irq_line, engine); } } @@ -2178,18 +1954,17 @@ static int irq_msix_channel_setup(struct xdma_dev *xdev) u32 vector; struct xdma_engine *engine; - if (!xdev) { - pr_err("dma engine NULL\n"); + if (unlikely(!xdev)) { + pr_err("xdev NULL.\n"); return -EINVAL; } - if (!xdev->msix_enabled) return 0; j = xdev->h2c_channel_max; engine = xdev->engine_h2c; for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) vector = pci_irq_vector(xdev->pdev, i); #else vector = xdev->entry[i].vector; @@ -2207,7 +1982,7 @@ static int irq_msix_channel_setup(struct xdma_dev *xdev) engine = xdev->engine_c2h; for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) vector = pci_irq_vector(xdev->pdev, j); #else vector = xdev->entry[j].vector; @@ -2231,20 +2006,19 @@ static void irq_msix_user_teardown(struct xdma_dev *xdev) int i; int j; - if (!xdev) { - pr_err("Invalid xdev\n"); + if (unlikely(!xdev)) { + pr_err("xdev NULL.\n"); return; } if (!xdev->msix_enabled) return; - j = xdev->h2c_channel_max + xdev->c2h_channel_max; - prog_irq_msix_user(xdev, 1); + j = xdev->h2c_channel_max + xdev->c2h_channel_max; for (i = 0; i < xdev->user_max; i++, j++) { -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; @@ -2262,16 +2036,16 @@ static int irq_msix_user_setup(struct xdma_dev *xdev) /* vectors set in probe_scan_for_msi() */ for (i = 0; i < xdev->user_max; i++, j++) { -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; #endif rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, - &xdev->user_irq[i]); + &xdev->user_irq[i]); if (rv) { - pr_info("user %d couldn't use IRQ#%d, %d\n", i, vector, - rv); + pr_info("user %d couldn't use IRQ#%d, %d\n", + i, vector, rv); break; } pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, @@ -2281,7 +2055,7 @@ static int irq_msix_user_setup(struct xdma_dev *xdev) /* If any errors occur, free IRQs that were successfully requested */ if (rv) { for (i--, j--; i >= 0; i--, j--) { -#if KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) u32 vector = pci_irq_vector(xdev->pdev, j); #else u32 vector = xdev->entry[j].vector; @@ -2318,23 +2092,22 @@ static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) dbg_init("Legacy Interrupt register value = %d\n", val); if (val > 1) { val--; - w = (val << 24) | (val << 16) | (val << 8) | val; + w = (val << 24) | (val << 16) | (val << 8)| val; /* Program IRQ Block Channel vactor and IRQ Block User vector - * with Legacy interrupt value - */ - reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + * with Legacy interrupt value */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user write_register(w, reg, 0x2080); write_register(w, reg + 0x4, 0x2084); write_register(w, reg + 0x8, 0x2088); write_register(w, reg + 0xC, 0x208C); - reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block write_register(w, reg, 0x20A0); write_register(w, reg + 0x4, 0x20A4); } xdev->irq_line = (int)pdev->irq; rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, - xdev); + xdev); if (rv) dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); else @@ -2360,7 +2133,6 @@ static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) if (xdev->msix_enabled) { int rv = irq_msix_channel_setup(xdev); - if (rv) return rv; rv = irq_msix_user_setup(xdev); @@ -2381,23 +2153,23 @@ static void dump_desc(struct xdma_desc *desc_virt) { int j; u32 *p = (u32 *)desc_virt; - static char *const field_name[] = { "magic|extra_adjacent|control", - "bytes", - "src_addr_lo", - "src_addr_hi", - "dst_addr_lo", - "dst_addr_hi", - "next_addr", - "next_addr_pad" }; + static char * const field_name[] = { "magic|extra_adjacent|control", + "bytes", + "src_addr_lo", + "src_addr_hi", + "dst_addr_lo", + "dst_addr_hi", + "next_addr", + "next_addr_pad"}; char *dummy; /* remove warning about unused variable when debug printing is off */ dummy = field_name[0]; for (j = 0; j < 8; j += 1) { - pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", (uintptr_t)p, - (uintptr_t)p & 15, (int)*p, le32_to_cpu(*p), - field_name[j]); + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", + (uintptr_t)p, (uintptr_t)p & 15, (int)*p, + le32_to_cpu(*p), field_name[j]); p++; } pr_info("\n"); @@ -2412,8 +2184,8 @@ static void transfer_dump(struct xdma_transfer *transfer) transfer, transfer->state, transfer->flags, transfer->dir, transfer->len, transfer->last_in_request); - pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", transfer, - transfer->desc_num, (u64)transfer->desc_bus, + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", + transfer, transfer->desc_num, (u64)transfer->desc_bus, transfer->desc_adjacent); for (i = 0; i < transfer->desc_num; i += 1) dump_desc(desc_virt + i); @@ -2437,10 +2209,12 @@ static int transfer_desc_init(struct xdma_transfer *transfer, int count) struct xdma_desc *desc_virt = transfer->desc_virt; dma_addr_t desc_bus = transfer->desc_bus; int i; + int adj = count - 1; + int extra_adj; + u32 temp_control; - if (count > XDMA_TRANSFER_MAX_DESC) { - pr_err("Engine cannot transfer more than %d descriptors\n", - XDMA_TRANSFER_MAX_DESC); + if (unlikely(count > XDMA_TRANSFER_MAX_DESC)) { + pr_err("xfer 0x%p, too many desc 0x%x.\n", transfer, count); return -EINVAL; } @@ -2453,14 +2227,31 @@ static int transfer_desc_init(struct xdma_transfer *transfer, int count) desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); desc_virt[i].bytes = cpu_to_le32(0); - desc_virt[i].control = cpu_to_le32(DESC_MAGIC); + + /* any adjacent descriptors? */ + if (adj > 0) { + extra_adj = adj - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + + adj--; + } else { + extra_adj = 0; + } + + temp_control = DESC_MAGIC | (extra_adj << 8); + + desc_virt[i].control = cpu_to_le32(temp_control); } /* { i = number - 1 } */ /* zero the last descriptor next pointer */ desc_virt[i].next_lo = cpu_to_le32(0); desc_virt[i].next_hi = cpu_to_le32(0); desc_virt[i].bytes = cpu_to_le32(0); - desc_virt[i].control = cpu_to_le32(DESC_MAGIC); + + temp_control = DESC_MAGIC; + + desc_virt[i].control = cpu_to_le32(temp_control); return 0; } @@ -2474,13 +2265,13 @@ static int transfer_desc_init(struct xdma_transfer *transfer, int count) * @second_bus bus address of second descriptor */ static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, - dma_addr_t second_bus) + dma_addr_t second_bus) { /* * remember reserved control in first descriptor, but zero * extra_adjacent! */ - u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; + u32 control = le32_to_cpu(first->control) & 0x00FFC0FFUL; /* second descriptor given? */ if (second) { /* @@ -2507,13 +2298,14 @@ static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) { /* remember reserved and control bits */ - u32 control = le32_to_cpu(desc->control) & 0xffffc0ffUL; + u32 control = le32_to_cpu(desc->control) & 0xFFFFC0FFUL; if (next_adjacent) next_adjacent = next_adjacent - 1; if (next_adjacent > MAX_EXTRA_ADJ) next_adjacent = MAX_EXTRA_ADJ; control |= (next_adjacent << 8); + /* write control and next_adjacent */ desc->control = cpu_to_le32(control); } @@ -2524,45 +2316,28 @@ static int xdma_desc_control_set(struct xdma_desc *first, u32 control_field) /* remember magic and adjacent number */ u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); - if (control_field & ~(LS_BYTE_MASK)) { - pr_err("Invalid control field\n"); + if (unlikely(control_field & ~(LS_BYTE_MASK))) { + pr_err("control_field bad 0x%x.\n", control_field); return -EINVAL; } /* merge adjacent and control field */ control |= control_field; /* write control and next_adjacent */ first->control = cpu_to_le32(control); + return 0; } /* xdma_desc_clear -- Clear bits in control field of a descriptor. */ -static int xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) { /* remember magic and adjacent number */ u32 control = le32_to_cpu(first->control); - if (clear_mask & ~(LS_BYTE_MASK)) { - pr_err("Invalid clear mask\n"); - return -EINVAL; - } - /* merge adjacent and control field */ control &= (~clear_mask); /* write control and next_adjacent */ first->control = cpu_to_le32(control); - return 0; -} - -/* xdma_desc_done - recycle cache-coherent linked list of descriptors. - * - * @dev Pointer to pci_dev - * @number Number of descriptors to be allocated - * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list - * @desc_bus Bus address of first descriptor in list - */ -static inline void xdma_desc_done(struct xdma_desc *desc_virt, int count) -{ - memset(desc_virt, 0, count * sizeof(struct xdma_desc)); } /* xdma_desc() - Fill a descriptor with the transfer details @@ -2577,7 +2352,7 @@ static inline void xdma_desc_done(struct xdma_desc *desc_virt, int count) * Does not modify the next pointer */ static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, - u64 ep_addr, int len, int dir) + u64 ep_addr, int len, int dir) { /* transfer length */ desc->bytes = cpu_to_le32(len); @@ -2601,41 +2376,34 @@ static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, /* * should hold the engine->lock; */ -static int transfer_abort(struct xdma_engine *engine, - struct xdma_transfer *transfer) +static void transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) { struct xdma_transfer *head; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (!transfer) { - pr_err("Invalid DMA transfer\n"); - return -EINVAL; + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); + return; } - - if (transfer->desc_num == 0) { - pr_err("%s void descriptors in the transfer list\n", - engine->name); - return -EINVAL; + if (unlikely(!transfer || (transfer->desc_num == 0))) { + pr_err("engine %s, xfer 0x%p, desc 0.\n", + engine->name, transfer); + return; } pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", transfer, transfer->desc_num, engine->desc_dequeued); head = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); + entry); if (head == transfer) list_del(engine->transfer_list.next); - else + else pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", engine->name, transfer, head); if (transfer->state == TRANSFER_STATE_SUBMITTED) transfer->state = TRANSFER_STATE_ABORTED; - return 0; } /* transfer_queue() - Queue a DMA transfer on the engine @@ -2646,39 +2414,29 @@ static int transfer_abort(struct xdma_engine *engine, * Takes and releases the engine spinlock */ static int transfer_queue(struct xdma_engine *engine, - struct xdma_transfer *transfer) + struct xdma_transfer *transfer) { int rv = 0; struct xdma_transfer *transfer_started; struct xdma_dev *xdev; unsigned long flags; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (!engine->xdev) { - pr_err("Invalid xdev\n"); + if (unlikely(!engine || !engine->xdev)) { + pr_err("bad engine 0x%p, xdev 0x%p.\n", + engine, engine ? engine->xdev : NULL); return -EINVAL; } - - if (!transfer) { - pr_err("%s Invalid DMA transfer\n", engine->name); - return -EINVAL; - } - - if (transfer->desc_num == 0) { - pr_err("%s void descriptors in the transfer list\n", - engine->name); + if (unlikely(!transfer || (transfer->desc_num == 0))) { + pr_err("engine %s, xfer 0x%p, desc 0.\n", + engine->name, transfer); return -EINVAL; } - dbg_tfr("%s (transfer=0x%p).\n", __func__, transfer); + dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); xdev = engine->xdev; if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { - pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", xdev, - transfer); + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", + xdev, transfer); return -EBUSY; } @@ -2704,12 +2462,9 @@ static int transfer_queue(struct xdma_engine *engine, /* engine is idle? */ if (!engine->running) { /* start engine */ - dbg_tfr("%s(): starting %s engine.\n", __func__, engine->name); + dbg_tfr("transfer_queue(): starting %s engine.\n", + engine->name); transfer_started = engine_start(engine); - if (!transfer_started) { - pr_err("Failed to start dma engine\n"); - goto shutdown; - } dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", transfer, engine->name, transfer_started); } else { @@ -2732,9 +2487,10 @@ static void engine_alignments(struct xdma_engine *engine) u32 address_bits; w = read_register(&engine->regs->alignments); - dbg_init("engine %p name %s alignments=0x%08x\n", engine, engine->name, - (int)w); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, + engine->name, (int)w); + /* RTO - add some macros to extract these fields */ align_bytes = (w & 0x00ff0000U) >> 16; granularity_bytes = (w & 0x0000ff00U) >> 8; address_bits = (w & 0x000000ffU); @@ -2762,72 +2518,61 @@ static void engine_free_resource(struct xdma_engine *engine) /* Release memory use for descriptor writebacks */ if (engine->poll_mode_addr_virt) { dbg_sg("Releasing memory for descriptor writeback\n"); - dma_free_coherent(&xdev->pdev->dev, sizeof(struct xdma_poll_wb), - engine->poll_mode_addr_virt, - engine->poll_mode_bus); + dma_free_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); dbg_sg("Released memory for descriptor writeback\n"); engine->poll_mode_addr_virt = NULL; } if (engine->desc) { dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", - dev_name(&xdev->pdev->dev), engine->name, engine->desc, - engine->desc_bus); + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); dma_free_coherent(&xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * - sizeof(struct xdma_desc), - engine->desc, engine->desc_bus); + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); engine->desc = NULL; } if (engine->cyclic_result) { - dma_free_coherent( - &xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result), + dma_free_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), engine->cyclic_result, engine->cyclic_result_bus); engine->cyclic_result = NULL; } } -static int engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) { - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } - - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; + if (unlikely(!xdev || !engine)) { + pr_err("xdev 0x%p, engine 0x%p.\n", xdev, engine); + return; } dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); /* Disable interrupts to stop processing new events during shutdown */ write_register(0x0, &engine->regs->interrupt_enable_mask, - (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); if (enable_credit_mp && engine->streaming && - engine->dir == DMA_FROM_DEVICE) { + engine->dir == DMA_FROM_DEVICE) { u32 reg_value = (0x1 << engine->channel) << 16; - struct sgdma_common_regs *reg = - (struct sgdma_common_regs - *)(xdev->bar[xdev->config_bar_idx] + - (0x6 * TARGET_SPACING)); + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); write_register(reg_value, ®->credit_mode_enable_w1c, 0); } - if (poll_mode) - xdma_thread_remove_work(engine); - /* Release memory use for descriptor writebacks */ engine_free_resource(engine); memset(engine, 0, sizeof(struct xdma_engine)); /* Decrement the number of engines available */ xdev->engines_num--; - return 0; } /** @@ -2837,51 +2582,42 @@ static int engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) */ struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) { - int rv; struct xdma_transfer *transfer = 0; - int size = engine->xdma_perf->transfer_size; /* transfers on queue? */ if (!list_empty(&engine->transfer_list)) { /* pick first transfer on the queue (was submitted to engine) */ transfer = list_entry(engine->transfer_list.next, - struct xdma_transfer, entry); - if (!transfer) { - pr_err("(engine=%s) has void transfer in queue.\n", - engine->name); - return NULL; - } - rv = xdma_engine_stop(engine); - if (rv < 0) { - pr_err("Failed to stop engine\n"); - return NULL; - } + struct xdma_transfer, entry); + + xdma_engine_stop(engine); engine->running = 0; - if (transfer->cyclic) { + if (transfer && transfer->cyclic) { if (engine->xdma_perf) dbg_perf("Stopping perf transfer on %s\n", - engine->name); + engine->name); else dbg_perf("Stopping cyclic transfer on %s\n", - engine->name); + engine->name); + /* free up the buffer allocated for perf run */ if (engine->perf_buf_virt) dma_free_coherent(&engine->xdev->pdev->dev, - size, engine->perf_buf_virt, - engine->perf_buf_bus); + engine->xdma_perf->transfer_size, + engine->perf_buf_virt, + engine->perf_buf_bus); engine->perf_buf_virt = NULL; list_del(&transfer->entry); } else { dbg_sg("(engine=%p) running transfer is not cyclic\n", - engine); + engine); } } else { dbg_sg("(engine=%p) found not running transfer.\n", engine); } return transfer; } -EXPORT_SYMBOL_GPL(engine_cyclic_stop); static int engine_writeback_setup(struct xdma_engine *engine) { @@ -2889,34 +2625,35 @@ static int engine_writeback_setup(struct xdma_engine *engine) struct xdma_dev *xdev; struct xdma_poll_wb *writeback; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || !engine->xdev)) { + pr_err("engine 0x%p, xdev NULL.\n", engine); return -EINVAL; } - xdev = engine->xdev; - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } + /* + * RTO - doing the allocation per engine is wasteful since a full page + * is allocated each time - better to allocate one page for the whole + * device during probe() and set per-engine offsets here + */ writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; writeback->completed_desc_count = 0; dbg_init("Setting writeback location to 0x%llx for engine %p", - engine->poll_mode_bus, engine); + engine->poll_mode_bus, engine); w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); - write_register(w, &engine->regs->poll_mode_wb_lo, - (unsigned long)(&engine->regs->poll_mode_wb_lo) - - (unsigned long)(&engine->regs)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); - write_register(w, &engine->regs->poll_mode_wb_hi, - (unsigned long)(&engine->regs->poll_mode_wb_hi) - - (unsigned long)(&engine->regs)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); return 0; } + /* engine_create() - Create an SG DMA engine bookkeeping data structure * * An SG DMA engine consists of the resources for a single-direction transfer @@ -2934,8 +2671,8 @@ static int engine_init_regs(struct xdma_engine *engine) int rv = 0; write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, - (unsigned long)(&engine->regs->control_w1c) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); engine_alignments(engine); @@ -2951,31 +2688,34 @@ static int engine_init_regs(struct xdma_engine *engine) rv = engine_writeback_setup(engine); if (rv) { dbg_init("%s descr writeback setup failed.\n", - engine->name); + engine->name); goto fail_wb; } } else { /* enable the relevant completion interrupts */ reg_value |= XDMA_CTRL_IE_DESC_STOPPED; reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) + reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; } /* Apply engine configurations */ write_register(reg_value, &engine->regs->interrupt_enable_mask, - (unsigned long)(&engine->regs->interrupt_enable_mask) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); engine->interrupt_enable_mask_value = reg_value; /* only enable credit mode for AXI-ST C2H */ if (enable_credit_mp && engine->streaming && - engine->dir == DMA_FROM_DEVICE) { + engine->dir == DMA_FROM_DEVICE) { + struct xdma_dev *xdev = engine->xdev; u32 reg_value = (0x1 << engine->channel) << 16; - struct sgdma_common_regs *reg = - (struct sgdma_common_regs - *)(xdev->bar[xdev->config_bar_idx] + - (0x6 * TARGET_SPACING)); + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); write_register(reg_value, ®->credit_mode_enable_w1s, 0); } @@ -2991,9 +2731,8 @@ static int engine_alloc_resource(struct xdma_engine *engine) struct xdma_dev *xdev = engine->xdev; engine->desc = dma_alloc_coherent(&xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * - sizeof(struct xdma_desc), - &engine->desc_bus, GFP_KERNEL); + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); if (!engine->desc) { pr_warn("dev %s, %s pre-alloc desc OOM.\n", dev_name(&xdev->pdev->dev), engine->name); @@ -3001,25 +2740,24 @@ static int engine_alloc_resource(struct xdma_engine *engine) } if (poll_mode) { - engine->poll_mode_addr_virt = - dma_alloc_coherent(&xdev->pdev->dev, - sizeof(struct xdma_poll_wb), - &engine->poll_mode_bus, GFP_KERNEL); + engine->poll_mode_addr_virt = dma_alloc_coherent( + &xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); if (!engine->poll_mode_addr_virt) { - pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", dev_name(&xdev->pdev->dev), engine->name); goto err_out; } } if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { - engine->cyclic_result = dma_alloc_coherent( - &xdev->pdev->dev, - XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_result), + engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), &engine->cyclic_result_bus, GFP_KERNEL); if (!engine->cyclic_result) { - pr_warn("%s, %s pre-alloc result OOM.\n", + pr_warn("%s, %s pre-alloc result OOM.\n", dev_name(&xdev->pdev->dev), engine->name); goto err_out; } @@ -3033,7 +2771,7 @@ static int engine_alloc_resource(struct xdma_engine *engine) } static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, - int offset, enum dma_data_direction dir, int channel) + int offset, enum dma_data_direction dir, int channel) { int rv; u32 val; @@ -3055,9 +2793,9 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, /* register address */ engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + - SGDMA_OFFSET_FROM_CHANNEL; + SGDMA_OFFSET_FROM_CHANNEL; val = read_register(&engine->regs->identifier); - if (val & 0x8000U) + if (val & 0x8000U) engine->streaming = 1; /* remember SG DMA direction */ @@ -3067,7 +2805,7 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, engine->streaming ? "ST" : "MM"); dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, - (int)engine->irq_bitmask); + (int)engine->irq_bitmask); /* initialize the deferred work for transfer completion */ INIT_WORK(&engine->work, engine_service_work); @@ -3086,167 +2824,67 @@ static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, if (rv) return rv; - if (poll_mode) - xdma_thread_add_work(engine); - return 0; } /* transfer_destroy() - free transfer */ static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) { - /* free descriptors */ - xdma_desc_done(xfer->desc_virt, xfer->desc_num); + /* free descriptors */ + memset(xfer->desc_virt, 0, xfer->desc_num * sizeof(struct xdma_desc)); if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { - struct sg_table *sgt = xfer->sgt; + struct sg_table *sgt = xfer->sgt; if (sgt->nents) { pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, - xfer->dir); + xfer->dir); sgt->nents = 0; } } } static int transfer_build(struct xdma_engine *engine, - struct xdma_request_cb *req, struct xdma_transfer *xfer, - unsigned int desc_max) + struct xdma_request_cb *req, unsigned int desc_max) { + struct xdma_transfer *xfer = &req->xfer; struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); int i = 0; int j = 0; - dma_addr_t bus = xfer->res_bus; for (; i < desc_max; i++, j++, sdesc++) { dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", - i + req->sw_desc_idx, req->sw_desc_cnt, sdesc->addr, - sdesc->len, req->ep_addr); + i + req->sw_desc_idx, req->sw_desc_cnt, + sdesc->addr, sdesc->len, req->ep_addr); /* fill in descriptor entry j with transfer details */ xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, - sdesc->len, xfer->dir); + sdesc->len, xfer->dir); xfer->len += sdesc->len; /* for non-inc-add mode don't increment ep_addr */ if (!engine->non_incr_addr) req->ep_addr += sdesc->len; - - if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { - memset(xfer->res_virt + j, 0, - sizeof(struct xdma_result)); - xfer->desc_virt[j].src_addr_lo = - cpu_to_le32(PCI_DMA_L(bus)); - xfer->desc_virt[j].src_addr_hi = - cpu_to_le32(PCI_DMA_H(bus)); - bus += sizeof(struct xdma_result); - } - } - req->sw_desc_idx += desc_max; + req->sw_desc_idx += desc_max; return 0; } - -static int transfer_init(struct xdma_engine *engine, - struct xdma_request_cb *req, struct xdma_transfer *xfer) +static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) { + struct xdma_transfer *xfer = &req->xfer; unsigned int desc_max = min_t(unsigned int, req->sw_desc_cnt - req->sw_desc_idx, XDMA_TRANSFER_MAX_DESC); - unsigned int desc_align = 0; int i = 0; int last = 0; u32 control; - unsigned long flags; - - memset(xfer, 0, sizeof(*xfer)); - - /* lock the engine state */ - spin_lock_irqsave(&engine->lock, flags); - /* initialize wait queue */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) - init_swait_queue_head(&xfer->wq); -#else - init_waitqueue_head(&xfer->wq); -#endif - - /* remember direction of transfer */ - xfer->dir = engine->dir; - xfer->desc_virt = engine->desc + engine->desc_idx; - xfer->res_virt = engine->cyclic_result + engine->desc_idx; - xfer->desc_bus = engine->desc_bus + - (sizeof(struct xdma_desc) * engine->desc_idx); - xfer->res_bus = engine->cyclic_result_bus + - (sizeof(struct xdma_result) * engine->desc_idx); - xfer->desc_index = engine->desc_idx; - - /* TODO: Need to handle desc_used >= XDMA_TRANSFER_MAX_DESC */ - - if ((engine->desc_idx + desc_max) >= XDMA_TRANSFER_MAX_DESC) - desc_max = XDMA_TRANSFER_MAX_DESC - engine->desc_idx; - - transfer_desc_init(xfer, desc_max); - - dbg_sg("xfer= %p transfer->desc_bus = 0x%llx.\n", - xfer, (u64)xfer->desc_bus); - transfer_build(engine, req, xfer, desc_max); - - /* - * Contiguous descriptors cannot cross PAGE boundary - * The 1st descriptor may start in the middle of the page, - * calculate the 1st block of adj desc accordingly - */ - desc_align = 128 - (engine->desc_idx % 128) - 1; - if (desc_align > (desc_max - 1)) - desc_align = desc_max - 1; - - xfer->desc_adjacent = desc_align; - - /* terminate last descriptor */ - last = desc_max - 1; - /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ - control = XDMA_DESC_STOPPED; - control |= XDMA_DESC_EOP; - control |= XDMA_DESC_COMPLETED; - xdma_desc_control_set(xfer->desc_virt + last, control); - - xfer->desc_num = desc_max; - engine->desc_idx = (engine->desc_idx + desc_max) % - XDMA_TRANSFER_MAX_DESC; - engine->desc_used += desc_max; - - /* fill in adjacent numbers */ - for (i = 0; i < xfer->desc_num && desc_align; i++, desc_align--) - xdma_desc_adjacent(xfer->desc_virt + i, desc_align); - - for (; i < xfer->desc_num; i++) - xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); - - spin_unlock_irqrestore(&engine->lock, flags); - return 0; -} - - -static int transfer_init_cyclic(struct xdma_engine *engine, - struct xdma_request_cb *req, - struct xdma_transfer *xfer) -{ - unsigned int desc_max = - min_t(unsigned int, req->sw_desc_cnt - req->sw_desc_idx, - XDMA_TRANSFER_MAX_DESC); - int i = 0; - u32 control; int rv; memset(xfer, 0, sizeof(*xfer)); /* initialize wait queue */ -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - init_swait_queue_head(&xfer->wq); -#else init_waitqueue_head(&xfer->wq); -#endif /* remember direction of transfer */ xfer->dir = engine->dir; @@ -3255,27 +2893,25 @@ static int transfer_init_cyclic(struct xdma_engine *engine, xfer->desc_bus = engine->desc_bus; rv = transfer_desc_init(xfer, desc_max); - if (rv < 0) { - pr_err("Failed to initialize descriptors\n"); + if (rv < 0) return rv; - } - + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); - transfer_build(engine, req, xfer, desc_max); + transfer_build(engine, req, desc_max); + /* terminate last descriptor */ + last = desc_max - 1; + xdma_desc_link(xfer->desc_virt + last, 0, 0); /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ control = XDMA_DESC_STOPPED; control |= XDMA_DESC_EOP; control |= XDMA_DESC_COMPLETED; - rv = xdma_desc_control_set(xfer->desc_virt + desc_max - 1, control); - if (rv < 0) { - pr_err("Failed to set desc control\n"); + rv = xdma_desc_control_set(xfer->desc_virt + last, control); + if (rv < 0) return rv; - } - xfer->desc_num = desc_max; - xfer->desc_adjacent = 1; + xfer->desc_num = xfer->desc_adjacent = desc_max; dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); /* fill in adjacent numbers */ @@ -3291,13 +2927,13 @@ static void sgt_dump(struct sg_table *sgt) int i; struct scatterlist *sg = sgt->sgl; - pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", sgt, sgt->sgl, sgt->nents, - sgt->orig_nents); + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", + sgt, sgt->sgl, sgt->nents, sgt->orig_nents); for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) - pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", i, sg, - sg_page(sg), sg->offset, sg->length, sg_dma_address(sg), - sg_dma_len(sg)); + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", + i, sg, sg_page(sg), sg->offset, sg->length, + sg_dma_address(sg), sg_dma_len(sg)); } static void xdma_request_cb_dump(struct xdma_request_cb *req) @@ -3308,8 +2944,9 @@ static void xdma_request_cb_dump(struct xdma_request_cb *req) req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); sgt_dump(req->sgt); for (i = 0; i < req->sw_desc_cnt; i++) - pr_info("%d/%u, 0x%llx, %u.\n", i, req->sw_desc_cnt, - req->sdesc[i].addr, req->sdesc[i].len); + pr_info("%d/%u, 0x%llx, %u.\n", + i, req->sw_desc_cnt, req->sdesc[i].addr, + req->sdesc[i].len); } #endif @@ -3322,11 +2959,11 @@ static void xdma_request_free(struct xdma_request_cb *req) kfree(req); } -static struct xdma_request_cb *xdma_request_alloc(unsigned int sdesc_nr) +static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) { struct xdma_request_cb *req; unsigned int size = sizeof(struct xdma_request_cb) + - sdesc_nr * sizeof(struct sw_desc); + sdesc_nr * sizeof(struct sw_desc); req = kzalloc(size, GFP_KERNEL); if (!req) { @@ -3342,8 +2979,8 @@ static struct xdma_request_cb *xdma_request_alloc(unsigned int sdesc_nr) return req; } -static struct xdma_request_cb *xdma_init_request(struct sg_table *sgt, - u64 ep_addr) +static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, + u64 ep_addr) { struct xdma_request_cb *req; struct scatterlist *sg = sgt->sgl; @@ -3351,427 +2988,67 @@ static struct xdma_request_cb *xdma_init_request(struct sg_table *sgt, int extra = 0; int i, j = 0; - for (i = 0; i < max; i++, sg = sg_next(sg)) { + for (i = 0; i < max; i++, sg = sg_next(sg)) { unsigned int len = sg_dma_len(sg); if (unlikely(len > desc_blen_max)) extra += (len + desc_blen_max - 1) / desc_blen_max; } - dbg_tfr("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); +//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); - max += extra; - req = xdma_request_alloc(max); - if (!req) - return NULL; - - req->sgt = sgt; - req->ep_addr = ep_addr; - - for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { - unsigned int tlen = sg_dma_len(sg); - dma_addr_t addr = sg_dma_address(sg); - - req->total_len += tlen; - while (tlen) { - req->sdesc[j].addr = addr; - if (tlen > desc_blen_max) { - req->sdesc[j].len = desc_blen_max; - addr += desc_blen_max; - tlen -= desc_blen_max; - } else { - req->sdesc[j].len = tlen; - tlen = 0; - } - j++; - } - } - - if (j > max) { - pr_err("Cannot transfer more than supported length %d\n", - desc_blen_max); - xdma_request_free(req); - return NULL; - } - req->sw_desc_cnt = j; -#ifdef __LIBXDMA_DEBUG__ - xdma_request_cb_dump(req); -#endif - return req; -} - -ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, - struct sg_table *sgt, bool dma_mapped, int timeout_ms) -{ - struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; - struct xdma_engine *engine; - int rv = 0, tfer_idx = 0, i; - ssize_t done = 0; - struct scatterlist *sg = sgt->sgl; - int nents; - enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; - struct xdma_request_cb *req = NULL; - struct xdma_result *result; - - if (!dev_hndl) - return -EINVAL; - - if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) - return -EINVAL; - - if (write == 1) { - if (channel >= xdev->h2c_channel_max) { - pr_err("H2C channel %d >= %d.\n", channel, - xdev->h2c_channel_max); - return -EINVAL; - } - engine = &xdev->engine_h2c[channel]; - } else if (write == 0) { - if (channel >= xdev->c2h_channel_max) { - pr_err("C2H channel %d >= %d.\n", channel, - xdev->c2h_channel_max); - return -EINVAL; - } - engine = &xdev->engine_c2h[channel]; - } - - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); - return -EINVAL; - } - - xdev = engine->xdev; - if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { - pr_info("xdev 0x%p, offline.\n", xdev); - return -EBUSY; - } - - /* check the direction */ - if (engine->dir != dir) { - pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", engine, - engine->name, channel, write, engine->dir, dir); - return -EINVAL; - } - - if (!dma_mapped) { - nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); - if (!nents) { - pr_info("map sgl failed, sgt 0x%p.\n", sgt); - return -EIO; - } - sgt->nents = nents; - } else { - if (!sgt->nents) { - pr_err("sg table has invalid number of entries 0x%p.\n", - sgt); - return -EIO; - } - } - - req = xdma_init_request(sgt, ep_addr); - if (!req) { - rv = -ENOMEM; - goto unmap_sgl; - } - - dbg_tfr("%s, len %u sg cnt %u.\n", engine->name, req->total_len, - req->sw_desc_cnt); - - sg = sgt->sgl; - nents = req->sw_desc_cnt; - mutex_lock(&engine->desc_lock); - - while (nents) { - unsigned long flags; - struct xdma_transfer *xfer; - - /* build transfer */ - rv = transfer_init(engine, req, &req->tfer[0]); - if (rv < 0) { - mutex_unlock(&engine->desc_lock); - goto unmap_sgl; - } - xfer = &req->tfer[0]; - - if (!dma_mapped) - xfer->flags = XFER_FLAG_NEED_UNMAP; - - /* last transfer for the given request? */ - nents -= xfer->desc_num; - if (!nents) { - xfer->last_in_request = 1; - xfer->sgt = sgt; - } - - dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", xfer->len, - req->ep_addr, done, req->sw_desc_idx, req->sw_desc_cnt); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); -#endif - - rv = transfer_queue(engine, xfer); - if (rv < 0) { - mutex_unlock(&engine->desc_lock); - pr_info("unable to submit %s, %d.\n", engine->name, rv); - goto unmap_sgl; - } - - /* - * When polling, determine how many descriptors have been queued - * on the engine to determine the writeback value expected - */ - if (poll_mode) { - unsigned int desc_count; - - spin_lock_irqsave(&engine->lock, flags); - desc_count = xfer->desc_num; - spin_unlock_irqrestore(&engine->lock, flags); - dbg_tfr("%s poll desc_count=%d\n", engine->name, - desc_count); - rv = engine_service_poll(engine, desc_count); - if (rv < 0) { - mutex_unlock(&engine->desc_lock); - pr_err("Failed to service polling\n"); - goto unmap_sgl; - } - - } else { - xlx_wait_event_interruptible_timeout( - xfer->wq, - (xfer->state != TRANSFER_STATE_SUBMITTED), - msecs_to_jiffies(timeout_ms)); - } - - spin_lock_irqsave(&engine->lock, flags); - - switch (xfer->state) { - case TRANSFER_STATE_COMPLETED: - spin_unlock_irqrestore(&engine->lock, flags); - - result = xfer->res_virt; - - dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", - xfer, xfer->len, req->ep_addr - xfer->len, - done); - - /* For C2H streaming use writeback results */ - if (engine->streaming && - engine->dir == DMA_FROM_DEVICE) { - for (i = 0; i < xfer->desc_num; i++) - done += result[i].length; - } else - done += xfer->len; - - rv = 0; - break; - case TRANSFER_STATE_FAILED: - pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", xfer, - xfer->len, req->ep_addr - xfer->len); - spin_unlock_irqrestore(&engine->lock, flags); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -EIO; - break; - default: - /* transfer can still be in-flight */ - pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", - xfer, xfer->len, xfer->state, req->ep_addr); - rv = engine_status_read(engine, 0, 1); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - } else if (rv == 0) { - //engine_status_dump(engine); - rv = transfer_abort(engine, xfer); - if (rv < 0) { - pr_err("Failed to stop engine\n"); - } else if (rv == 0) { - rv = xdma_engine_stop(engine); - if (rv < 0) - pr_err("Failed to stop engine\n"); - } - } - spin_unlock_irqrestore(&engine->lock, flags); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -ERESTARTSYS; - break; - } - - engine->desc_used -= xfer->desc_num; - transfer_destroy(xdev, xfer); - - /* use multiple transfers per request if we could not fit - * all data within single descriptor chain. - */ - tfer_idx++; - - if (rv < 0) { - mutex_unlock(&engine->desc_lock); - goto unmap_sgl; - } - } /* while (sg) */ - mutex_unlock(&engine->desc_lock); - -unmap_sgl: - if (!dma_mapped && sgt->nents) { - pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); - sgt->nents = 0; - } - - if (req) - xdma_request_free(req); - - if (rv < 0) - return rv; - - return done; -} -EXPORT_SYMBOL_GPL(xdma_xfer_submit); - -ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, - bool write, u64 ep_addr, struct sg_table *sgt, - bool dma_mapped, int timeout_ms) -{ - - struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; - struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; - struct xdma_engine *engine; - int rv = 0, tfer_idx = 0; - ssize_t done = 0; - int nents; - enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; - struct xdma_request_cb *req = NULL; - struct xdma_transfer *xfer; - int i; - struct xdma_result *result; - - if (write == 1) { - if (channel >= xdev->h2c_channel_max) { - pr_warn("H2C channel %d >= %d.\n", - channel, xdev->h2c_channel_max); - return -EINVAL; - } - engine = &xdev->engine_h2c[channel]; - } else if (write == 0) { - if (channel >= xdev->c2h_channel_max) { - pr_warn("C2H channel %d >= %d.\n", - channel, xdev->c2h_channel_max); - return -EINVAL; - } - engine = &xdev->engine_c2h[channel]; - } else { - pr_warn("write %d, exp. 0|1.\n", write); - return -EINVAL; - } - - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); - return -EINVAL; - } - - xdev = engine->xdev; - req = cb->req; - - nents = req->sw_desc_cnt; - while (nents) { - xfer = &req->tfer[tfer_idx]; - nents -= xfer->desc_num; - switch (xfer->state) { - case TRANSFER_STATE_COMPLETED: - dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", - xfer, xfer->len, req->ep_addr - xfer->len, - done); - - result = xfer->res_virt; - /* For C2H streaming use writeback results */ - if (engine->streaming && - engine->dir == DMA_FROM_DEVICE) { - for (i = 0; i < xfer->desc_num; i++) - done += result[i].length; - } else - done += xfer->len; - - rv = 0; - break; - case TRANSFER_STATE_FAILED: - pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", - xfer, xfer->len, req->ep_addr - xfer->len); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -EIO; - break; - default: - /* transfer can still be in-flight */ - pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", - xfer, xfer->len, xfer->state, req->ep_addr); - engine_status_read(engine, 0, 1); - engine_status_dump(engine); - transfer_abort(engine, xfer); - - xdma_engine_stop(engine); - -#ifdef __LIBXDMA_DEBUG__ - transfer_dump(xfer); - sgt_dump(sgt); -#endif - rv = -ERESTARTSYS; - break; - } - - transfer_destroy(xdev, xfer); - engine->desc_used -= xfer->desc_num; + max += extra; + req = xdma_request_alloc(max); + if (!req) + return NULL; - tfer_idx++; + req->sgt = sgt; + req->ep_addr = ep_addr; - if (rv < 0) - goto unmap_sgl; - } /* while (sg) */ + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + unsigned int tlen = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); -unmap_sgl: - if (!dma_mapped && sgt->nents) { - pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); - sgt->nents = 0; + req->total_len += tlen; + while (tlen) { + req->sdesc[j].addr = addr; + if (tlen > desc_blen_max) { + req->sdesc[j].len = desc_blen_max; + addr += desc_blen_max; + tlen -= desc_blen_max; + } else { + req->sdesc[j].len = tlen; + tlen = 0; + } + j++; + if (j > max) + break; + } } - if (req) + if (unlikely(j > max)) { + pr_err("too many sdesc %d > %d\n", j, max); +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif xdma_request_free(req); + return NULL; + } - return done; - + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; } -EXPORT_SYMBOL_GPL(xdma_xfer_completion); - -ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, - bool write, u64 ep_addr, struct sg_table *sgt, - bool dma_mapped, int timeout_ms) +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) { struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; struct xdma_engine *engine; - struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl; - int rv = 0, tfer_idx = 0; + int rv = 0; + ssize_t done = 0; struct scatterlist *sg = sgt->sgl; int nents; enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; @@ -3802,14 +3079,9 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, return -EINVAL; } - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", - engine->name, engine->magic); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } @@ -3834,10 +3106,9 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, } sgt->nents = nents; } else { - if (!sgt->nents) { - pr_err("sg table has invalid number of entries 0x%p.\n", - sgt); - return -EIO; + if (unlikely(!sgt->nents)) { + pr_err("%s, sgt NOT dma_mapped.\n", engine->name); + return -EINVAL; } } @@ -3847,39 +3118,24 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, goto unmap_sgl; } - //used when doing completion. - req->cb = cb; - cb->req = req; dbg_tfr("%s, len %u sg cnt %u.\n", engine->name, req->total_len, req->sw_desc_cnt); sg = sgt->sgl; nents = req->sw_desc_cnt; - while (nents) { + mutex_lock(&engine->desc_lock); + while (nents) { + unsigned long flags; struct xdma_transfer *xfer; - /* one transfer at a time */ - xfer = &req->tfer[tfer_idx]; - /* build transfer */ - rv = transfer_init(engine, req, xfer); + /* build transfer */ + rv = transfer_init(engine, req); if (rv < 0) { - pr_info("transfer_init failed\n"); - - if (!dma_mapped && sgt->nents) { - pci_unmap_sg(xdev->pdev, sgt->sgl, - sgt->orig_nents, dir); - sgt->nents = 0; - } - - /* Transfer failed return BUSY */ - if (cb->io_done) - cb->io_done((unsigned long)cb, -EBUSY); - - goto rel_req; + mutex_unlock(&engine->desc_lock); + goto unmap_sgl; } - - xfer->cb = cb; + xfer = &req->xfer; if (!dma_mapped) xfer->flags = XFER_FLAG_NEED_UNMAP; @@ -3891,9 +3147,9 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, xfer->sgt = sgt; } - dbg_tfr("xfer %p, len %u, ep 0x%llx, sg %u/%u. nents = %d\n", - xfer, xfer->len, req->ep_addr, req->sw_desc_idx, - req->sw_desc_cnt, nents); + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", + xfer->len, req->ep_addr, done, req->sw_desc_idx, + req->sw_desc_cnt); #ifdef __LIBXDMA_DEBUG__ transfer_dump(xfer); @@ -3901,17 +3157,78 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, rv = transfer_queue(engine, xfer); if (rv < 0) { + mutex_unlock(&engine->desc_lock); pr_info("unable to submit %s, %d.\n", engine->name, rv); goto unmap_sgl; } - /* use multiple transfers per request if we could not fit all - * data within single descriptor chain. + /* + * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected */ - tfer_idx++; - } + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("%s poll desc_count=%d\n", + engine->name, desc_count); + rv = engine_service_poll(engine, desc_count); + + } else { + rv = wait_event_interruptible_timeout(xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch(xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, done); + done += xfer->len; + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + //engine_status_dump(engine); + transfer_abort(engine, xfer); - return -EIOCBQUEUED; + xdma_engine_stop(engine); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + + if (rv < 0) + break; + } /* while (sg) */ + mutex_unlock(&engine->desc_lock); unmap_sgl: if (!dma_mapped && sgt->nents) { @@ -3919,30 +3236,30 @@ ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, sgt->nents = 0; } -rel_req: if (req) xdma_request_free(req); - return rv; -} -EXPORT_SYMBOL_GPL(xdma_xfer_submit_nowait); + if (rv < 0) + return rv; + return done; +} int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) { - u32 max_consistent_size = XDMA_PERF_NUM_DESC * 32 * 1024; /* 4MB */ + u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ struct xdma_transfer *transfer; u64 ep_addr = 0; - int num_desc_in_a_loop = XDMA_PERF_NUM_DESC; + int num_desc_in_a_loop = 128; int size_in_desc = engine->xdma_perf->transfer_size; int size = size_in_desc * num_desc_in_a_loop; + int free_desc = 0; int i; int rv = -ENOMEM; - unsigned char free_desc = 0; - if (size_in_desc > max_consistent_size) { - pr_err("%s max consistent size %d is more than supported %d\n", - engine->name, size_in_desc, max_consistent_size); + if (unlikely(size_in_desc > max_consistent_size)) { + pr_err("%s, size too big %d > %u.\n", + engine->name, size_in_desc, max_consistent_size); return -EINVAL; } @@ -3951,23 +3268,20 @@ int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) num_desc_in_a_loop = size / size_in_desc; } - engine->perf_buf_virt = dma_alloc_coherent(&xdev->pdev->dev, - size_in_desc, - &engine->perf_buf_bus, - GFP_KERNEL); - if (!engine->perf_buf_virt) { - pr_err("dev %s, %s DMA allocation OOM.\n", - dev_name(&xdev->pdev->dev), engine->name); - return rv; + engine->perf_buf_virt = dma_alloc_coherent(&xdev->pdev->dev, size, + &engine->perf_buf_bus, GFP_KERNEL); + if (unlikely(!engine->perf_buf_virt)) { + pr_err("engine %s perf buf OOM.\n", engine->name); + return -ENOMEM; } /* allocate transfer data structure */ transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); - if (!transfer) { - pr_err("dev %s, %s transfer request OOM.\n", - dev_name(&xdev->pdev->dev), engine->name); - goto err_engine_transfer; + if (unlikely(!transfer)) { + pr_err("engine %s transfer OOM.\n", engine->name); + goto free_buffer; } + /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ transfer->dir = engine->dir; /* set number of descriptors */ @@ -3975,90 +3289,81 @@ int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) /* allocate descriptor list */ if (!engine->desc) { - engine->desc = dma_alloc_coherent( - &xdev->pdev->dev, + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, num_desc_in_a_loop * sizeof(struct xdma_desc), &engine->desc_bus, GFP_KERNEL); - if (!engine->desc) { - pr_err("%s DMA memory allocation for descriptors failed\n", - engine->name); - goto err_engine_desc; + if (unlikely(!engine->desc)) { + pr_err("%s desc OOM.\n", engine->name); + goto free_xfer; } dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", - dev_name(&xdev->pdev->dev), engine->name, engine->desc, - engine->desc_bus); + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); free_desc = 1; } transfer->desc_virt = engine->desc; transfer->desc_bus = engine->desc_bus; rv = transfer_desc_init(transfer, transfer->desc_num); - if (rv < 0) { - pr_err("Failed to initialize descriptors\n"); - goto err_dma_desc; - } + if (rv < 0) + goto free_desc; dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); for (i = 0; i < transfer->desc_num; i++) { struct xdma_desc *desc = transfer->desc_virt + i; - dma_addr_t rc_bus_addr = engine->perf_buf_bus; + dma_addr_t rc_bus_addr = engine->perf_buf_bus + + size_in_desc * i; /* fill in descriptor entry with transfer details */ xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, - engine->dir); + engine->dir); } /* stop engine and request interrupt on last descriptor */ rv = xdma_desc_control_set(transfer->desc_virt, 0); if (rv < 0) { - pr_err("Failed to set desc control\n"); - goto err_dma_desc; + pr_err("%s: Failed to set desc control\n", engine->name); + goto free_desc; } + /* create a linked loop */ xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, - transfer->desc_virt, transfer->desc_bus); + transfer->desc_virt, transfer->desc_bus); transfer->cyclic = 1; /* initialize wait queue */ -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - init_swait_queue_head(&transfer->wq); -#else init_waitqueue_head(&transfer->wq); -#endif - - //printk("=== Descriptor print for PERF\n"); - //transfer_dump(transfer); dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", - engine->dir ? "write (to dev)" : "read (from dev)"); + engine->dir ? "write (to dev)" : "read (from dev)"); rv = transfer_queue(engine, transfer); - if (rv < 0) { - pr_err("Failed to queue transfer\n"); - goto err_dma_desc; - } + if (rv < 0) + goto free_desc; + return 0; -err_dma_desc: +free_desc: if (free_desc && engine->desc) dma_free_coherent(&xdev->pdev->dev, num_desc_in_a_loop * sizeof(struct xdma_desc), engine->desc, engine->desc_bus); engine->desc = NULL; -err_engine_desc: - if (transfer) + +free_xfer: + if (transfer) { list_del(&transfer->entry); - kfree(transfer); - transfer = NULL; -err_engine_transfer: + kfree(transfer); + } + +free_buffer: if (engine->perf_buf_virt) dma_free_coherent(&xdev->pdev->dev, size_in_desc, - engine->perf_buf_virt, engine->perf_buf_bus); + engine->perf_buf_virt, engine->perf_buf_bus); engine->perf_buf_virt = NULL; return rv; } -EXPORT_SYMBOL_GPL(xdma_performance_submit); static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) { @@ -4066,15 +3371,15 @@ static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) struct xdma_dev *xdev; struct xdma_engine *engine; - if (!pdev) { - pr_err("Invalid pdev\n"); + if (unlikely(!pdev)) { + pr_err("pdev NULL.\n"); return NULL; } /* allocate zeroed device book keeping structure */ xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); if (!xdev) { - pr_info("OOM, xdma_dev.\n"); + pr_info("xdev OOM.\n"); return NULL; } spin_lock_init(&xdev->lock); @@ -4103,13 +3408,8 @@ static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) spin_lock_init(&engine->lock); mutex_init(&engine->desc_lock); INIT_LIST_HEAD(&engine->transfer_list); -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - init_swait_queue_head(&engine->shutdown_wq); - init_swait_queue_head(&engine->xdma_perf_wq); -#else init_waitqueue_head(&engine->shutdown_wq); init_waitqueue_head(&engine->xdma_perf_wq); -#endif } engine = xdev->engine_c2h; @@ -4117,13 +3417,8 @@ static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) spin_lock_init(&engine->lock); mutex_init(&engine->desc_lock); INIT_LIST_HEAD(&engine->transfer_list); -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - init_swait_queue_head(&engine->shutdown_wq); - init_swait_queue_head(&engine->xdma_perf_wq); -#else init_waitqueue_head(&engine->shutdown_wq); init_waitqueue_head(&engine->xdma_perf_wq); -#endif } return xdev; @@ -4133,13 +3428,8 @@ static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) { int rv; - if (!xdev) { - pr_err("Invalid xdev\n"); - return -EINVAL; - } - - if (!pdev) { - pr_err("Invalid pdev\n"); + if (unlikely(!xdev || !pdev)) { + pr_err("xdev 0x%p, pdev 0x%p.\n", xdev, pdev); return -EINVAL; } @@ -4159,8 +3449,8 @@ static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) static int set_dma_mask(struct pci_dev *pdev) { - if (!pdev) { - pr_err("Invalid pdev\n"); + if (unlikely(!pdev)) { + pr_err("pdev NULL.\n"); return -EINVAL; } @@ -4188,13 +3478,13 @@ static int set_dma_mask(struct pci_dev *pdev) return 0; } -static int get_engine_channel_id(struct engine_regs *regs) +static u32 get_engine_channel_id(struct engine_regs *regs) { - int value; + u32 value; - if (!regs) { - pr_err("Invalid engine registers\n"); - return -EINVAL; + if (unlikely(!regs)) { + pr_err("regs NULL.\n"); + return 0xFFFFFFFF; } value = read_register(®s->identifier); @@ -4202,13 +3492,13 @@ static int get_engine_channel_id(struct engine_regs *regs) return (value & 0x00000f00U) >> 8; } -static int get_engine_id(struct engine_regs *regs) +static u32 get_engine_id(struct engine_regs *regs) { - int value; + u32 value; - if (!regs) { - pr_err("Invalid engine registers\n"); - return -EINVAL; + if (unlikely(!regs)) { + pr_err("regs NULL.\n"); + return 0xFFFFFFFF; } value = read_register(®s->identifier); @@ -4219,10 +3509,9 @@ static void remove_engines(struct xdma_dev *xdev) { struct xdma_engine *engine; int i; - int rv; - if (!xdev) { - pr_err("Invalid xdev\n"); + if (unlikely(!xdev)) { + pr_err("xdev NULL.\n"); return; } @@ -4231,9 +3520,7 @@ static void remove_engines(struct xdma_dev *xdev) engine = &xdev->engine_h2c[i]; if (engine->magic == MAGIC_ENGINE) { dbg_sg("Remove %s, %d", engine->name, i); - rv = engine_destroy(xdev, engine); - if (rv < 0) - pr_err("Failed to destroy H2C engine %d\n", i); + engine_destroy(xdev, engine); dbg_sg("%s, %d removed", engine->name, i); } } @@ -4242,16 +3529,14 @@ static void remove_engines(struct xdma_dev *xdev) engine = &xdev->engine_c2h[i]; if (engine->magic == MAGIC_ENGINE) { dbg_sg("Remove %s, %d", engine->name, i); - rv = engine_destroy(xdev, engine); - if (rv < 0) - pr_err("Failed to destroy C2H engine %d\n", i); + engine_destroy(xdev, engine); dbg_sg("%s, %d removed", engine->name, i); } } } static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, - int channel) + int channel) { struct engine_regs *regs; int offset = channel * CHANNEL_SPACING; @@ -4263,8 +3548,7 @@ static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, /* register offset for the engine */ /* read channels at 0x0000, write channels at 0x1000, - * channels at 0x100 interval - */ + * channels at 0x100 interval */ if (dir == DMA_TO_DEVICE) { engine_id_expected = XDMA_ID_H2C; engine = &xdev->engine_h2c[channel]; @@ -4279,23 +3563,24 @@ static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, channel_id = get_engine_channel_id(regs); if ((engine_id != engine_id_expected) || (channel_id != channel)) { - dbg_init( - "%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x,exp 0x%x,0x%x, SKIP.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, offset, - engine_id, channel_id, engine_id_expected, - channel_id != channel); + dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," + "exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel, offset, engine_id, channel_id, + engine_id_expected, channel_id != channel); return -EINVAL; } dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, offset, - engine_id, channel_id); + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, + offset, engine_id, channel_id); /* allocate and initialize engine */ rv = engine_init(engine, xdev, offset, dir, channel); if (rv != 0) { pr_info("failed to create AXI %s %d engine.\n", - dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel); + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel); return rv; } @@ -4307,8 +3592,8 @@ static int probe_engines(struct xdma_dev *xdev) int i; int rv = 0; - if (!xdev) { - pr_err("Invalid xdev\n"); + if (unlikely(!xdev)) { + pr_err("xdev NULL.\n"); return -EINVAL; } @@ -4330,7 +3615,7 @@ static int probe_engines(struct xdma_dev *xdev) return 0; } -#if KERNEL_VERSION(3, 5, 0) <= LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) static void pci_enable_capability(struct pci_dev *pdev, int cap) { pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, cap); @@ -4351,7 +3636,7 @@ static void pci_enable_capability(struct pci_dev *pdev, int cap) #endif void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, - int *h2c_channel_max, int *c2h_channel_max) + int *h2c_channel_max, int *c2h_channel_max) { struct xdma_dev *xdev = NULL; int rv = 0; @@ -4373,12 +3658,12 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) xdev->user_max = MAX_USER_IRQ; if (xdev->h2c_channel_max == 0 || - xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; if (xdev->c2h_channel_max == 0 || - xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; - + rv = pci_enable_device(pdev); if (rv) { dbg_init("pci_enable_device() failed, %d.\n", rv); @@ -4465,7 +3750,6 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, kfree(xdev); return NULL; } -EXPORT_SYMBOL_GPL(xdma_device_open); void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) { @@ -4479,11 +3763,11 @@ void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) return; - dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", pdev, - xdev); + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", + pdev, xdev); if (xdev->pdev != pdev) { dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", - (unsigned long)xdev->pdev, (unsigned long)pdev); + (unsigned long)xdev->pdev, (unsigned long)pdev); } channel_interrupts_disable(xdev, ~0); @@ -4510,14 +3794,12 @@ void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) kfree(xdev); } -EXPORT_SYMBOL_GPL(xdma_device_close); void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) { struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; struct xdma_engine *engine; int i; - int rv; if (!dev_hndl) return; @@ -4533,16 +3815,13 @@ void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) unsigned long flags; engine = &xdev->engine_h2c[i]; - + if (engine->magic == MAGIC_ENGINE) { spin_lock_irqsave(&engine->lock, flags); engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; - rv = xdma_engine_stop(engine); - if (rv < 0) - pr_err("Failed to stop engine\n"); - else - engine->running = 0; + xdma_engine_stop(engine); + engine->running = 0; spin_unlock_irqrestore(&engine->lock, flags); } } @@ -4555,11 +3834,8 @@ void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) spin_lock_irqsave(&engine->lock, flags); engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; - rv = xdma_engine_stop(engine); - if (rv < 0) - pr_err("Failed to stop engine\n"); - else - engine->running = 0; + xdma_engine_stop(engine); + engine->running = 0; spin_unlock_irqrestore(&engine->lock, flags); } } @@ -4572,7 +3848,6 @@ void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) pr_info("xdev 0x%p, done.\n", xdev); } -EXPORT_SYMBOL_GPL(xdma_device_offline); void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) { @@ -4587,9 +3862,9 @@ void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) return; - pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); - for (i = 0; i < xdev->h2c_channel_max; i++) { + for (i = 0; i < xdev->h2c_channel_max; i++) { engine = &xdev->engine_h2c[i]; if (engine->magic == MAGIC_ENGINE) { engine_init_regs(engine); @@ -4599,7 +3874,7 @@ void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) } } - for (i = 0; i < xdev->c2h_channel_max; i++) { + for (i = 0; i < xdev->c2h_channel_max; i++) { engine = &xdev->engine_c2h[i]; if (engine->magic == MAGIC_ENGINE) { engine_init_regs(engine); @@ -4617,11 +3892,10 @@ void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) user_interrupts_enable(xdev, xdev->mask_irq_user); read_interrupts(xdev); } - + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); pr_info("xdev 0x%p, done.\n", xdev); } -EXPORT_SYMBOL_GPL(xdma_device_online); int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) { @@ -4636,10 +3910,9 @@ int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) pr_info("NOT implemented, 0x%p.\n", xdev); return -EINVAL; } -EXPORT_SYMBOL_GPL(xdma_device_restart); int xdma_user_isr_register(void *dev_hndl, unsigned int mask, - irq_handler_t handler, void *dev) + irq_handler_t handler, void *dev) { struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; int i; @@ -4663,7 +3936,6 @@ int xdma_user_isr_register(void *dev_hndl, unsigned int mask, return 0; } -EXPORT_SYMBOL_GPL(xdma_user_isr_register); int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) { @@ -4682,7 +3954,6 @@ int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) return 0; } -EXPORT_SYMBOL_GPL(xdma_user_isr_enable); int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) { @@ -4693,67 +3964,40 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) return -EINVAL; - + xdev->mask_irq_user &= ~mask; user_interrupts_disable(xdev, mask); read_interrupts(xdev); return 0; } -EXPORT_SYMBOL_GPL(xdma_user_isr_disable); - -#ifdef __LIBXDMA_MOD__ -static int __init xdma_base_init(void) -{ - pr_info("%s", version); - return 0; -} - -static void __exit xdma_base_exit(void) -{ - pr_info("%s", __func__); -} -module_init(xdma_base_init); -module_exit(xdma_base_exit); -#endif /* makes an existing transfer cyclic */ static void xdma_transfer_cyclic(struct xdma_transfer *transfer) { /* link last descriptor to first descriptor */ xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, - transfer->desc_virt, transfer->desc_bus); + transfer->desc_virt, transfer->desc_bus); /* remember transfer is cyclic */ transfer->cyclic = 1; } static int transfer_monitor_cyclic(struct xdma_engine *engine, - struct xdma_transfer *transfer, - int timeout_ms) + struct xdma_transfer *transfer, int timeout_ms) { struct xdma_result *result; int rc = 0; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (!transfer) { - pr_err("%s: xfer empty.\n", engine->name); + if (unlikely(!engine || !engine->cyclic_result || !transfer)) { + pr_err("engine 0x%p, cyclic_result 0x%p, xfer 0x%p.\n", + engine, engine->cyclic_result, transfer); return -EINVAL; } result = engine->cyclic_result; - if (!result) { - pr_err("%s Cyclic transfer resources not available.\n", - engine->name); - return -EINVAL; - } if (poll_mode) { - int i; - + int i ; for (i = 0; i < 5; i++) { rc = engine_service_poll(engine, 0); if (rc) { @@ -4761,46 +4005,33 @@ static int transfer_monitor_cyclic(struct xdma_engine *engine, engine->name, rc); rc = -ERESTARTSYS; } - if (result[engine->rx_head].status) { - rc = 0; - break; - } + if (result[engine->rx_head].status) + return 0; } } else { - if (enable_credit_mp) { + if (enable_credit_mp){ dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", engine->name, engine->rx_head, engine->rx_tail); - - rc = xlx_wait_event_interruptible_timeout( - transfer->wq, - (engine->rx_head != engine->rx_tail || - engine->rx_overrun), - msecs_to_jiffies(timeout_ms)); - + rc = wait_event_interruptible_timeout( transfer->wq, + (engine->rx_head!=engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", - engine->name, rc, engine->rx_head, + engine->name, rc, engine->rx_head, engine->rx_tail, engine->rx_overrun); } else { - rc = xlx_wait_event_interruptible_timeout( - transfer->wq, - engine->eop_found, - msecs_to_jiffies(timeout_ms)); - + rc = wait_event_interruptible_timeout( transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); dbg_tfr("%s: wait returns %d, eop_found %d.\n", engine->name, rc, engine->eop_found); } - /* condition evaluated to false after the timeout elapsed */ - if (rc == 0) - rc = -ETIME; - /* condition evaluated to true */ - else if (rc > 0) - rc = 0; } - return rc; + return 0; } -static struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) { struct scatterlist *sg = sgt->sgl; int i; @@ -4818,48 +4049,44 @@ static struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) } static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, - int head, char __user *buf, size_t count) + int head, char __user *buf, size_t count) { struct scatterlist *sg; int more = pkt_length; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (!buf) { - pr_err("%s invalid user buffer pointer\n", engine->name); + if (unlikely(!buf || !engine)) { + pr_err("engine 0x%p, buf 0x%p.\n", engine, buf); return -EINVAL; } - dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", engine->name, - pkt_length, head, engine->user_buffer_index); + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", + engine->name, pkt_length, head, engine->user_buffer_index); sg = sglist_index(&engine->cyclic_sgt, head); if (!sg) { - pr_info("%s, head %d OOR, sgl %u.\n", engine->name, head, - engine->cyclic_sgt.orig_nents); + pr_info("%s, head %d OOR, sgl %u.\n", + engine->name, head, engine->cyclic_sgt.orig_nents); return -EIO; } /* EOP found? Transfer anything from head to EOP */ while (more) { - unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; unsigned int blen = count - engine->user_buffer_index; int rv; if (copy > blen) copy = blen; - dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", engine->name, - head, sg, copy, engine->user_buffer_index); + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", + engine->name, head, sg, copy, + engine->user_buffer_index); rv = copy_to_user(&buf[engine->user_buffer_index], - page_address(sg_page(sg)), copy); + page_address(sg_page(sg)), copy); if (rv) { - pr_info("%s copy_to_user %u failed %d\n", engine->name, - copy, rv); + pr_info("%s copy_to_user %u failed %d\n", + engine->name, copy, rv); return -EIO; } @@ -4870,7 +4097,7 @@ static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, /* user buffer used up */ break; } - + head++; if (head >= CYCLIC_RX_PAGES_MAX) { head = 0; @@ -4894,30 +4121,24 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, int num_credit = 0; unsigned long flags; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || !engine->cyclic_result)) { + pr_err("engine 0x%p, cyclic_result NULL.\n", engine); return -EINVAL; } result = engine->cyclic_result; - if (!result) { - pr_err("%s Cyclic transfer resources not available.\n", - engine->name); - return -EINVAL; - } - spin_lock_irqsave(&engine->lock, flags); /* where the host currently is in the ring buffer */ head = engine->rx_head; /* iterate over newly received results */ - while ((engine->rx_head != engine->rx_tail || engine->rx_overrun) && - pkt_length < (int)count) { - WARN_ON(result[engine->rx_head].status == 0); + while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { + + WARN_ON(result[engine->rx_head].status==0); dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", - engine->name, engine->rx_head, + engine->name, engine->rx_head, result[engine->rx_head].status, result[engine->rx_head].length); @@ -4932,23 +4153,24 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, result[engine->rx_head].length, PAGE_SIZE); fault = 1; } else if (result[engine->rx_head].length == 0) { - pr_info("%s, result[%d].length 0x%x.\n", engine->name, - engine->rx_head, + pr_info("%s, result[%d].length 0x%x.\n", + engine->name, engine->rx_head, result[engine->rx_head].length); fault = 1; /* valid result */ } else { pkt_length += result[engine->rx_head].length; - num_credit++; + num_credit++; /* seen eop? */ //if (result[engine->rx_head].status & RX_STATUS_EOP) - if (result[engine->rx_head].status & RX_STATUS_EOP) { + if (result[engine->rx_head].status & RX_STATUS_EOP){ eop = 1; engine->eop_found = 1; } - dbg_tfr("%s, pkt_length=%d (%s)\n", engine->name, - pkt_length, eop ? "with EOP" : "no EOP yet"); + dbg_tfr("%s, pkt_length=%d (%s)\n", + engine->name, pkt_length, + eop ? "with EOP" : "no EOP yet"); } /* clear result */ result[engine->rx_head].status = 0; @@ -4957,20 +4179,21 @@ static int complete_cyclic(struct xdma_engine *engine, char __user *buf, engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; /* stop processing if a fault/eop was detected */ - if (fault || eop) + if (fault || eop){ break; + } } spin_unlock_irqrestore(&engine->lock, flags); if (fault) return -EIO; - + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); - engine->rx_overrun = 0; + engine->rx_overrun = 0; /* if copy is successful, release credits */ - if (rc > 0) - write_register(num_credit, &engine->sgdma_regs->credits, 0); + if(rc > 0) + write_register(num_credit,&engine->sgdma_regs->credits, 0); return rc; } @@ -4983,25 +4206,20 @@ ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, int rc_len = 0; struct xdma_transfer *transfer; - if (!engine) { - pr_err("dma engine NULL\n"); - return -EINVAL; - } - - if (engine->magic != MAGIC_ENGINE) { - pr_err("%s has invalid magic number %lx\n", engine->name, - engine->magic); + if (unlikely(!engine || (engine->magic != MAGIC_ENGINE))) { + pr_err("bad engine 0x%p, magic 0x%lx.\n", + engine, engine ? engine->magic : 0UL); return -EINVAL; } - transfer = &engine->cyclic_req->tfer[0]; - if (!transfer) { - pr_err("Invalid DMA transfer\n"); + if (unlikely(!engine->cyclic_req)) { + pr_err("engine %s, cyclic_req NULL.\n", engine->name); return -EINVAL; } + transfer = &engine->cyclic_req->xfer; - engine->user_buffer_index = 0; - + engine->user_buffer_index = 0; + do { rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); if (rc < 0) @@ -5011,16 +4229,13 @@ ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, return rc; rc_len += rc; - if (rc_len >= (int)count) - return rc_len; - i++; if (i > 10) break; } while (!engine->eop_found); - //Always reset EOP found indication regardless of credit mechanims - engine->eop_found = 0; + if(enable_credit_mp) + engine->eop_found = 0; return rc_len; } @@ -5062,17 +4277,17 @@ static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, for (i = 0; i < npages; i++, sg = sg_next(sg)) { struct page *pg = alloc_page(GFP_KERNEL); - if (!pg) { + if (!pg) { pr_info("%d/%u, page OOM.\n", i, npages); goto err_out; } if (pdev) { - dma_addr_t bus = - pci_map_page(pdev, pg, 0, PAGE_SIZE, dir); - if (unlikely(pci_dma_mapping_error(pdev, bus))) { - pr_info("%d/%u, page 0x%p map err.\n", i, - npages, pg); + dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, + dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", + i, npages, pg); __free_page(pg); goto err_out; } @@ -5088,7 +4303,7 @@ static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, err_out: sgt_free_with_pages(sgt, dir, pdev); - return -ENOMEM; + return -ENOMEM; } int xdma_cyclic_transfer_setup(struct xdma_engine *engine) @@ -5100,19 +4315,15 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) int i; int rc; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine || !engine->xdev)) { + pr_err("engine 0x%p, xdev NULL.\n", engine); return -EINVAL; } - xdev = engine->xdev; - if (!xdev) { - pr_err("Invalid DMA devie\n"); - return -EINVAL; - } if (engine->cyclic_req) { - dbg_tfr("%s: exclusive access already taken.\n", engine->name); + pr_info("%s: exclusive access already taken.\n", + engine->name); return -EBUSY; } @@ -5124,10 +4335,10 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) engine->eop_found = 0; rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, - engine->dir, xdev->pdev); + engine->dir, xdev->pdev); if (rc < 0) { - pr_info("%s cyclic pages %u OOM.\n", engine->name, - CYCLIC_RX_PAGES_MAX); + pr_info("%s cyclic pages %u OOM.\n", + engine->name, CYCLIC_RX_PAGES_MAX); goto err_out; } @@ -5142,34 +4353,27 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) xdma_request_cb_dump(engine->cyclic_req); #endif - xfer = &engine->cyclic_req->tfer[0]; - rc = transfer_init_cyclic(engine, engine->cyclic_req, xfer); + rc = transfer_init(engine, engine->cyclic_req); if (rc < 0) goto err_out; + xfer = &engine->cyclic_req->xfer; + /* replace source addresses with result write-back addresses */ memset(engine->cyclic_result, 0, - CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); bus = engine->cyclic_result_bus; - for (i = 0; i < xfer->desc_num; i++) { + for (i = 0; i < xfer->desc_num; i++) { xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); - xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); - bus += sizeof(struct xdma_result); - } + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } /* set control of all descriptors */ - for (i = 0; i < xfer->desc_num; i++) { - rc = xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); - if (rc < 0) { - pr_err("Failed to clear desc control\n"); - goto err_out; - } - rc = xdma_desc_control_set(xfer->desc_virt + i, - XDMA_DESC_EOP | XDMA_DESC_COMPLETED); - if (rc < 0) { - pr_err("Failed to set desc control\n"); - goto err_out; - } - } + for (i = 0; i < xfer->desc_num; i++) { + xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + } /* make this a cyclic transfer */ xdma_transfer_cyclic(xfer); @@ -5178,34 +4382,27 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) transfer_dump(xfer); #endif - if (enable_credit_mp) { - //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); + if (enable_credit_mp) write_register(128, &engine->sgdma_regs->credits, 0); - } - - /* start cyclic transfer */ - rc = transfer_queue(engine, xfer); - if (rc < 0) { - pr_err("Failed to queue transfer\n"); - goto err_out; - } - - xfer->last_in_request = 1; spin_unlock_irqrestore(&engine->lock, flags); - return 0; + /* start cyclic transfer */ + rc = transfer_queue(engine, xfer); + if (!rc) + return 0; + spin_lock_irqsave(&engine->lock, flags); /* unwind on errors */ err_out: if (engine->cyclic_req) { - xdma_request_free(engine->cyclic_req); + xdma_request_free(engine->cyclic_req); engine->cyclic_req = NULL; } - + if (engine->cyclic_sgt.orig_nents) { sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, - xdev->pdev); + xdev->pdev); engine->cyclic_sgt.orig_nents = 0; engine->cyclic_sgt.nents = 0; engine->cyclic_sgt.sgl = NULL; @@ -5218,10 +4415,8 @@ int xdma_cyclic_transfer_setup(struct xdma_engine *engine) static int cyclic_shutdown_polled(struct xdma_engine *engine) { - int rv; - - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return -EINVAL; } @@ -5229,58 +4424,36 @@ static int cyclic_shutdown_polled(struct xdma_engine *engine) dbg_tfr("Polling for shutdown completion\n"); do { - rv = engine_status_read(engine, 1, 0); - if (rv < 0) { - pr_err("Failed to read engine status\n"); - goto failure; - } + engine_status_read(engine, 1, 0); schedule(); } while (engine->status & XDMA_STAT_BUSY); if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { dbg_tfr("Engine has stopped\n"); - if (!list_empty(&engine->transfer_list)) { - rv = engine_transfer_dequeue(engine); - if (rv < 0) { - pr_err("Failed to dequeue transfer\n"); - goto failure; - } - } + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); - rv = engine_service_shutdown(engine); - if (rv < 0) { - pr_err("Failed to shutdown engine\n"); - goto failure; - } + engine_service_shutdown(engine); } -failure: + dbg_tfr("Shutdown completion polling done\n"); spin_unlock(&engine->lock); - return rv; + return 0; } static int cyclic_shutdown_interrupt(struct xdma_engine *engine) { int rc; - if (!engine) { - pr_err("dma engine NULL\n"); + if (unlikely(!engine)) { + pr_err("engine NULL.\n"); return -EINVAL; } - rc = xlx_wait_event_interruptible_timeout(engine->shutdown_wq, - !engine->running, - msecs_to_jiffies(10000)); - -#if 0 - if (rc) { - dbg_tfr("wait_event_interruptible=%d\n", rc); - return rc; - } -#endif - + rc = wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, msecs_to_jiffies(10000)); if (engine->running) { pr_info("%s still running?!, %d\n", engine->name, rc); return -EINVAL; @@ -5305,10 +4478,10 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) spin_lock_irqsave(&engine->lock, flags); if (transfer) { dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); - if (transfer != &engine->cyclic_req->tfer[0]) { + if (transfer != &engine->cyclic_req->xfer) { pr_info("%s unexpected transfer 0x%p/0x%p\n", engine->name, transfer, - &engine->cyclic_req->tfer[0]); + &engine->cyclic_req->xfer); } } /* allow engine to be serviced after stop request */ @@ -5319,7 +4492,6 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) rc = cyclic_shutdown_polled(engine); else rc = cyclic_shutdown_interrupt(engine); - if (rc < 0) { pr_err("Failed to shutdown cyclic transfers\n"); return rc; @@ -5335,7 +4507,7 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) if (engine->cyclic_sgt.orig_nents) { sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, - xdev->pdev); + xdev->pdev); engine->cyclic_sgt.orig_nents = 0; engine->cyclic_sgt.nents = 0; engine->cyclic_sgt.sgl = NULL; @@ -5349,7 +4521,7 @@ int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) { int rv; - unsigned long dst; + unsigned long dst; u32 w = XDMA_CTRL_NON_INCR_ADDR; dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); @@ -5358,15 +4530,13 @@ int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) if (rv == 0) { engine->non_incr_addr = !!dst; if (engine->non_incr_addr) - write_register( - w, &engine->regs->control_w1s, + write_register(w, &engine->regs->control_w1s, (unsigned long)(&engine->regs->control_w1s) - - (unsigned long)(&engine->regs)); + (unsigned long)(&engine->regs)); else - write_register( - w, &engine->regs->control_w1c, - (unsigned long)(&engine->regs->control_w1c) - - (unsigned long)(&engine->regs)); + write_register(w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); } engine_alignments(engine); diff --git a/sdk/linux_kernel_drivers/xdma/libxdma.h b/sdk/linux_kernel_drivers/xdma/libxdma.h old mode 100755 new mode 100644 index 9f889cd9f..1fbee5aaf --- a/sdk/linux_kernel_drivers/xdma/libxdma.h +++ b/sdk/linux_kernel_drivers/xdma/libxdma.h @@ -21,7 +21,6 @@ * Karen Xie * ******************************************************************************/ - #ifndef XDMA_LIB_H #define XDMA_LIB_H @@ -36,28 +35,26 @@ #include #include #include -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE -#include -#endif + /* - * if the config bar is fixed, the driver does not neeed to search through - * all of the bars + * if the config bar is fixed, the driver does not neeed to search through + * all of the bars */ //#define XDMA_CONFIG_BAR_NUM 1 /* Switch debug printing on/off */ -#define XDMA_DEBUG 0 +#define XDMA_DEBUG 0 /* SECTION: Preprocessor macros/constants */ -#define XDMA_BAR_NUM (6) +#define XDMA_BAR_NUM (6) /* maximum amount of register space to map */ -#define XDMA_BAR_SIZE (0x8000UL) +#define XDMA_BAR_SIZE (0x8000UL) /* Use this definition to poll several times between calls to schedule */ -#define NUM_POLLS_PER_SCHED 100 +#define NUM_POLLS_PER_SCHED 100 -#define XDMA_CHANNEL_NUM_MAX (4) +#define XDMA_CHANNEL_NUM_MAX (4) /* * interrupts per engine, rad2_vul.sv:237 * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), @@ -88,7 +85,6 @@ #define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) #define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) #define XDMA_CTRL_POLL_MODE_WB (1UL << 26) -#define XDMA_CTRL_STM_MODE_WB (1UL << 27) /* bits of the SG DMA status register */ #define XDMA_STAT_BUSY (1UL << 0) @@ -144,7 +140,7 @@ /* all combined */ #define XDMA_STAT_H2C_ERR_MASK \ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ - XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) #define XDMA_STAT_C2H_ERR_MASK \ (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ @@ -253,23 +249,6 @@ enum dev_capabilities { /* SECTION: Structure definitions */ -struct xdma_io_cb { - void __user *buf; - size_t len; - void *private; - unsigned int pages_nr; - struct sg_table sgt; - struct page **pages; - /** total data size */ - unsigned int count; - /** MM only, DDR/BRAM memory addr */ - u64 ep_addr; - /** write: if write to the device */ - struct xdma_request_cb *req; - u8 write:1; - void (*io_done)(unsigned long cb_hndl, int err); -}; - struct config_regs { u32 identifier; u32 reserved_1[4]; @@ -415,18 +394,11 @@ struct sw_desc { struct xdma_transfer { struct list_head entry; /* queue of non-completed transfers */ struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ - struct xdma_result *res_virt; /* virt addr of result, c2h streaming */ - dma_addr_t res_bus; /* bus addr for result descriptors */ dma_addr_t desc_bus; /* bus addr of the first descriptor */ int desc_adjacent; /* adjacent descriptors at desc_bus */ int desc_num; /* number of descriptors in transfer */ - int desc_index; /* index for 1st desc. in transfer */ enum dma_data_direction dir; -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - struct swait_queue_head wq; -#else wait_queue_head_t wq; /* wait queue for transfer completion */ -#endif enum transfer_state state; /* state of the transfer */ unsigned int flags; @@ -435,7 +407,6 @@ struct xdma_transfer { int last_in_request; /* flag if last within request */ unsigned int len; struct sg_table *sgt; - struct xdma_io_cb *cb; }; struct xdma_request_cb { @@ -443,10 +414,7 @@ struct xdma_request_cb { unsigned int total_len; u64 ep_addr; - /* Use two transfers in case single request needs to be split */ - struct xdma_transfer tfer[2]; - - struct xdma_io_cb *cb; + struct xdma_transfer xfer; unsigned int sw_desc_idx; unsigned int sw_desc_cnt; @@ -489,13 +457,14 @@ struct xdma_engine { /* Members applicable to AXI-ST C2H (cyclic) transfers */ struct xdma_result *cyclic_result; dma_addr_t cyclic_result_bus; /* bus addr for transfer */ - struct xdma_request_cb *cyclic_req; - struct sg_table cyclic_sgt; + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 *perf_buf_virt; dma_addr_t perf_buf_bus; /* bus address */ u8 eop_found; /* used only for cyclic(rx:c2h) */ - int eop_count; + int rx_tail; /* follows the HW */ int rx_head; /* where the SW reads from */ int rx_overrun; /* flag if overrun occured */ @@ -508,11 +477,7 @@ struct xdma_engine { dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ /* Members associated with interrupt mode support */ -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - struct swait_queue_head shutdown_wq; -#else wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ -#endif spinlock_t lock; /* protects concurrent access */ int prev_cpu; /* remember CPU# of (last) locker */ int msix_irq_line; /* MSI-X vector for this engine */ @@ -522,23 +487,10 @@ struct xdma_engine { struct mutex desc_lock; /* protects concurrent access */ dma_addr_t desc_bus; struct xdma_desc *desc; - int desc_idx; /* current descriptor index */ - int desc_used; /* total descriptors used */ /* for performance test support */ struct xdma_performance_ioctl *xdma_perf; /* perf test control */ -#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE - struct swait_queue_head xdma_perf_wq; -#else wait_queue_head_t xdma_perf_wq; /* Perf test sync */ -#endif - - struct xdma_kthread *cmplthp; - /* completion status thread list for the queue */ - struct list_head cmplthp_list; - /* pending work thread list */ - /* cpu attached to intr_work */ - unsigned int intr_work_cpu; }; struct xdma_user_irq { @@ -584,7 +536,7 @@ struct xdma_dev { int irq_line; /* flag if irq allocated successfully */ int msi_enabled; /* flag if msi was enabled for the device */ int msix_enabled; /* flag if msi-x was enabled for the device */ -#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) struct msix_entry entry[32]; /* msi-x vector/entry table */ #endif struct xdma_user_irq user_irq[16]; /* user IRQ management */ @@ -664,8 +616,8 @@ void get_perf_stats(struct xdma_engine *engine); int xdma_cyclic_transfer_setup(struct xdma_engine *engine); int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); -ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, - char __user *buf, size_t count, int timeout_ms); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count, int timeout_ms); int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); -int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count); + #endif /* XDMA_LIB_H */ diff --git a/sdk/linux_kernel_drivers/xdma/libxdma_api.h b/sdk/linux_kernel_drivers/xdma/libxdma_api.h index f652d7d9d..d4ed4ec50 100644 --- a/sdk/linux_kernel_drivers/xdma/libxdma_api.h +++ b/sdk/linux_kernel_drivers/xdma/libxdma_api.h @@ -82,10 +82,7 @@ void xdma_device_close(struct pci_dev *pdev, void *dev_handle); /* * xdma_device_restart - restart the fpga * @pdev: ptr to struct pci_dev - * TODO: - * may need more refining on the parameter list * return < 0 in case of error - * TODO: exact error code will be defined later */ int xdma_device_restart(struct pci_dev *pdev, void *dev_handle); @@ -106,7 +103,6 @@ int xdma_device_restart(struct pci_dev *pdev, void *dev_handle); * @name: to be passed to the handler, ignored if handler is NULL` * @dev: to be passed to the handler, ignored if handler is NULL` * return < 0 in case of error - * TODO: exact error code will be defined later */ int xdma_user_isr_register(void *dev_hndl, unsigned int mask, irq_handler_t handler, void *dev); @@ -116,7 +112,6 @@ int xdma_user_isr_register(void *dev_hndl, unsigned int mask, * @pdev: ptr to the the pci_dev struct * @mask: bitmask of user interrupts (0 ~ 15)to be registered * return < 0 in case of error - * TODO: exact error code will be defined later */ int xdma_user_isr_enable(void *dev_hndl, unsigned int mask); int xdma_user_isr_disable(void *dev_hndl, unsigned int mask); @@ -133,23 +128,8 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask); * @timeout: timeout in mili-seconds, *currently ignored * return # of bytes transfered or * < 0 in case of error - * TODO: exact error code will be defined later */ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, struct sg_table *sgt, bool dma_mapped, int timeout_ms); -ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr, - struct sg_table *sgt, bool dma_mapped, int timeout_ms); - - -ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr, - struct sg_table *sgt, bool dma_mapped, int timeout_ms); - - - -/////////////////////missing API//////////////////// - -//xdma_get_channle_state - if no interrupt on DMA hang is available -//xdma_channle_restart - #endif diff --git a/sdk/linux_kernel_drivers/xdma/version.h b/sdk/linux_kernel_drivers/xdma/version.h old mode 100755 new mode 100644 index 094a285fd..5ed57832d --- a/sdk/linux_kernel_drivers/xdma/version.h +++ b/sdk/linux_kernel_drivers/xdma/version.h @@ -27,7 +27,7 @@ #define DRV_MOD_MAJOR 2020 #define DRV_MOD_MINOR 1 -#define DRV_MOD_PATCHLEVEL 06 +#define DRV_MOD_PATCHLEVEL 01 #define DRV_MODULE_VERSION \ __stringify(DRV_MOD_MAJOR) "." \ diff --git a/sdk/linux_kernel_drivers/xdma/xdma_cdev.c b/sdk/linux_kernel_drivers/xdma/xdma_cdev.c index 3a540c82d..a5c3ac553 100644 --- a/sdk/linux_kernel_drivers/xdma/xdma_cdev.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_cdev.c @@ -527,7 +527,6 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) } xpdev_flag_set(xpdev, XDF_CDEV_SG); - /* ??? Bypass */ /* Initialize Bypass Character Device */ if (xdev->bypass_bar_idx > 0) { for (i = 0; i < xpdev->h2c_channel_max; i++) { @@ -615,28 +614,11 @@ int xdma_cdev_init(void) return -EINVAL; } - /* using kmem_cache_create to enable sequential cleanup */ - cdev_cache = kmem_cache_create("cdev_cache", - sizeof(struct cdev_async_io), 0, - SLAB_HWCACHE_ALIGN, NULL); - - if (!cdev_cache) { - pr_info("memory allocation for cdev_cache failed. OOM\n"); - return -ENOMEM; - } - - xdma_threads_create(num_online_cpus()); - return 0; } void xdma_cdev_cleanup(void) { - if (cdev_cache) - kmem_cache_destroy(cdev_cache); - if (g_xdma_class) class_destroy(g_xdma_class); - - xdma_threads_destroy(); } diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.c b/sdk/linux_kernel_drivers/xdma/xdma_mod.c old mode 100755 new mode 100644 index c9672069e..b9dbfcfe6 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.c +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.c @@ -50,6 +50,10 @@ MODULE_LICENSE("GPL v2"); static int xpdev_cnt; static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(0x10ee, 0x9048), }, + { PCI_DEVICE(0x10ee, 0x9044), }, + { PCI_DEVICE(0x10ee, 0x9042), }, + { PCI_DEVICE(0x10ee, 0x9041), }, { PCI_DEVICE(0x10ee, 0x903f), }, { PCI_DEVICE(0x10ee, 0x9038), }, { PCI_DEVICE(0x10ee, 0x9028), }, diff --git a/sdk/linux_kernel_drivers/xdma/xdma_mod.h b/sdk/linux_kernel_drivers/xdma/xdma_mod.h old mode 100755 new mode 100644 index 2bf25fe70..abea67ee0 --- a/sdk/linux_kernel_drivers/xdma/xdma_mod.h +++ b/sdk/linux_kernel_drivers/xdma/xdma_mod.h @@ -52,7 +52,6 @@ #include #include "libxdma.h" -#include "xdma_thread.h" #define MAGIC_ENGINE 0xEEEEEEEEUL #define MAGIC_DEVICE 0xDDDDDDDDUL @@ -104,19 +103,12 @@ struct xdma_pci_dev { void *data; }; -struct cdev_async_io { - struct kiocb *iocb; - struct xdma_io_cb *cb; - bool write; - bool cancel; - int cmpl_cnt; - int req_cnt; - spinlock_t lock; - struct work_struct wrk_itm; - struct cdev_async_io *next; - ssize_t res; - ssize_t res2; - int err_cnt; +struct xdma_io_cb { + void __user *buf; + size_t len; + unsigned int pages_nr; + struct sg_table sgt; + struct page **pages; }; #endif /* ifndef __XDMA_MODULE_H__ */ diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.c b/sdk/linux_kernel_drivers/xdma/xdma_thread.c deleted file mode 100644 index 8b57f14be..000000000 --- a/sdk/linux_kernel_drivers/xdma/xdma_thread.c +++ /dev/null @@ -1,345 +0,0 @@ -/******************************************************************************* - * - * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2020 Xilinx, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program. If not, see . - * - * The full GNU General Public License is included in this distribution in - * the file called "LICENSE". - * - * Karen Xie - * - ******************************************************************************/ - -#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ - -#include "xdma_thread.h" - -#include -#include - - -/* ********************* global variables *********************************** */ -static struct xdma_kthread *cs_threads; -static unsigned int thread_cnt; - - -/* ********************* static function definitions ************************ */ -static int xdma_thread_cmpl_status_pend(struct list_head *work_item) -{ - struct xdma_engine *engine = list_entry(work_item, struct xdma_engine, - cmplthp_list); - int pend = 0; - unsigned long flags; - - spin_lock_irqsave(&engine->lock, flags); - pend = !list_empty(&engine->transfer_list); - spin_unlock_irqrestore(&engine->lock, flags); - - return pend; -} - -static int xdma_thread_cmpl_status_proc(struct list_head *work_item) -{ - struct xdma_engine *engine; - struct xdma_transfer * transfer; - - engine = list_entry(work_item, struct xdma_engine, cmplthp_list); - transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, - entry); - engine_service_poll(engine, transfer->desc_num); - return 0; -} - - - -static inline int xthread_work_pending(struct xdma_kthread *thp) -{ - struct list_head *work_item, *next; - - /* any work items assigned to this thread? */ - if (list_empty(&thp->work_list)) - return 0; - - - /* any work item has pending work to do? */ - list_for_each_safe(work_item, next, &thp->work_list) { - if (thp->fpending && thp->fpending(work_item)) - return 1; - - } - return 0; -} - -static inline void xthread_reschedule(struct xdma_kthread *thp) -{ - if (thp->timeout) { - pr_debug_thread("%s rescheduling for %u seconds", - thp->name, thp->timeout); - wait_event_interruptible_timeout(thp->waitq, thp->schedule, - msecs_to_jiffies(thp->timeout)); - } else { - pr_debug_thread("%s rescheduling", thp->name); - wait_event_interruptible(thp->waitq, thp->schedule); - } -} - -static int xthread_main(void *data) -{ - struct xdma_kthread *thp = (struct xdma_kthread *)data; - - pr_debug_thread("%s UP.\n", thp->name); - - disallow_signal(SIGPIPE); - - if (thp->finit) - thp->finit(thp); - - - while (!kthread_should_stop()) { - - struct list_head *work_item, *next; - - pr_debug_thread("%s interruptible\n", thp->name); - - /* any work to do? */ - lock_thread(thp); - if (!xthread_work_pending(thp)) { - unlock_thread(thp); - xthread_reschedule(thp); - lock_thread(thp); - } - thp->schedule = 0; - - if (thp->work_cnt) { - pr_debug_thread("%s processing %u work items\n", - thp->name, thp->work_cnt); - /* do work */ - list_for_each_safe(work_item, next, &thp->work_list) { - thp->fproc(work_item); - } - } - unlock_thread(thp); - schedule(); - } - - pr_debug_thread("%s, work done.\n", thp->name); - - if (thp->fdone) - thp->fdone(thp); - - pr_debug_thread("%s, exit.\n", thp->name); - return 0; -} - - -int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id) -{ - int len; - int node; - - if (thp->task) { - pr_warn("kthread %s task already running?\n", thp->name); - return -EINVAL; - } - - len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id); - if (len < 0) - return -EINVAL; - - thp->id = id; - - spin_lock_init(&thp->lock); - INIT_LIST_HEAD(&thp->work_list); - init_waitqueue_head(&thp->waitq); - - node = cpu_to_node(thp->cpu); - pr_debug("node : %d\n", node); - - thp->task = kthread_create_on_node(xthread_main, (void *)thp, - node, "%s", thp->name); - if (IS_ERR(thp->task)) { - pr_err("kthread %s, create task failed: 0x%lx\n", - thp->name, (unsigned long)IS_ERR(thp->task)); - thp->task = NULL; - return -EFAULT; - } - - kthread_bind(thp->task, thp->cpu); - - pr_debug_thread("kthread 0x%p, %s, cpu %u, task 0x%p.\n", - thp, thp->name, thp->cpu, thp->task); - - wake_up_process(thp->task); - return 0; -} - - -int xdma_kthread_stop(struct xdma_kthread *thp) -{ - int rv; - - if (!thp->task) { - pr_debug_thread("kthread %s, already stopped.\n", thp->name); - return 0; - } - - thp->schedule = 1; - rv = kthread_stop(thp->task); - if (rv < 0) { - pr_warn("kthread %s, stop err %d.\n", thp->name, rv); - return rv; - } - - pr_debug_thread("kthread %s, 0x%p, stopped.\n", thp->name, thp->task); - thp->task = NULL; - - return 0; -} - - - -void xdma_thread_remove_work(struct xdma_engine *engine) -{ - struct xdma_kthread *cmpl_thread; - unsigned long flags; - - spin_lock_irqsave(&engine->lock, flags); - cmpl_thread = engine->cmplthp; - engine->cmplthp = NULL; - -// pr_debug("%s removing from thread %s, %u.\n", -// descq->conf.name, cmpl_thread ? cmpl_thread->name : "?", -// cpu_idx); - - spin_unlock_irqrestore(&engine->lock, flags); - -#if 0 - if (cpu_idx < cpu_count) { - spin_lock(&qcnt_lock); - per_cpu_qcnt[cpu_idx]--; - spin_unlock(&qcnt_lock); - } -#endif - - if (cmpl_thread) { - lock_thread(cmpl_thread); - list_del(&engine->cmplthp_list); - cmpl_thread->work_cnt--; - unlock_thread(cmpl_thread); - } -} - -void xdma_thread_add_work(struct xdma_engine *engine) -{ - struct xdma_kthread *thp = cs_threads; - unsigned int v = 0; - int i, idx = thread_cnt; - unsigned long flags; - - /* Polled mode only */ - for (i = 0; i < thread_cnt; i++, thp++) { - lock_thread(thp); - if (idx == thread_cnt) { - v = thp->work_cnt; - idx = i; - } else if (!thp->work_cnt) { - idx = i; - unlock_thread(thp); - break; - } else if (thp->work_cnt < v) - idx = i; - unlock_thread(thp); - } - - thp = cs_threads + idx; - lock_thread(thp); - list_add_tail(&engine->cmplthp_list, &thp->work_list); - engine->intr_work_cpu = idx; - thp->work_cnt++; - unlock_thread(thp); - - pr_info("%s 0x%p assigned to cmpl status thread %s,%u.\n", - engine->name, engine, thp->name, thp->work_cnt); - - - spin_lock_irqsave(&engine->lock, flags); - engine->cmplthp = thp; - spin_unlock_irqrestore(&engine->lock, flags); -} - -int xdma_threads_create(unsigned int num_threads) -{ - struct xdma_kthread *thp; - int rv; - int cpu; - - if (thread_cnt) { - pr_warn("threads already created!"); - return 0; - } - - pr_info("xdma_threads_create\n"); - - cs_threads = kzalloc(num_threads * sizeof(struct xdma_kthread), - GFP_KERNEL); - if (!cs_threads) - return -ENOMEM; - - /* N dma writeback monitoring threads */ - thp = cs_threads; - for_each_online_cpu(cpu) { - pr_debug("index %d cpu %d online\n", thread_cnt, cpu); - thp->cpu = cpu; - thp->timeout = 0; - thp->fproc = xdma_thread_cmpl_status_proc; - thp->fpending = xdma_thread_cmpl_status_pend; - rv = xdma_kthread_start(thp, "cmpl_status_th", thread_cnt); - if (rv < 0) - goto cleanup_threads; - - thread_cnt++; - if (thread_cnt == num_threads) - break; - thp++; - } - - return 0; - -cleanup_threads: - kfree(cs_threads); - cs_threads = NULL; - thread_cnt = 0; - - return rv; -} - -void xdma_threads_destroy(void) -{ - int i; - struct xdma_kthread *thp; - - if (!thread_cnt) - return; - - /* N dma writeback monitoring threads */ - thp = cs_threads; - for (i = 0; i < thread_cnt; i++, thp++) - if (thp->fproc) - xdma_kthread_stop(thp); - - kfree(cs_threads); - cs_threads = NULL; - thread_cnt = 0; -} diff --git a/sdk/linux_kernel_drivers/xdma/xdma_thread.h b/sdk/linux_kernel_drivers/xdma/xdma_thread.h deleted file mode 100644 index bb5c15fd5..000000000 --- a/sdk/linux_kernel_drivers/xdma/xdma_thread.h +++ /dev/null @@ -1,152 +0,0 @@ -/******************************************************************************* - * - * Xilinx XDMA IP Core Linux Driver - * Copyright(c) 2015 - 2020 Xilinx, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program. If not, see . - * - * The full GNU General Public License is included in this distribution in - * the file called "LICENSE". - * - * Karen Xie - * - ******************************************************************************/ - -#ifndef __XDMA_KTHREAD_H__ -#define __XDMA_KTHREAD_H__ -/** - * @file - * @brief This file contains the declarations for xdma kernel threads - * - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include "libxdma.h" - -#ifdef DEBUG_THREADS -#define lock_thread(thp) \ - do { \ - pr_debug("locking thp %s ...\n", (thp)->name); \ - spin_lock(&(thp)->lock); \ - } while (0) - -#define unlock_thread(thp) \ - do { \ - pr_debug("unlock thp %s ...\n", (thp)->name); \ - spin_unlock(&(thp)->lock); \ - } while (0) - -#define xdma_kthread_wakeup(thp) \ - do { \ - pr_info("signaling thp %s ...\n", (thp)->name); \ - wake_up_process((thp)->task); \ - } while (0) - -#define pr_debug_thread(fmt, ...) pr_info(fmt, __VA_ARGS__) - -#else -/** lock thread macro */ -#define lock_thread(thp) spin_lock(&(thp)->lock) -/** un lock thread macro */ -#define unlock_thread(thp) spin_unlock(&(thp)->lock) -#define xdma_kthread_wakeup(thp) \ - do { \ - thp->schedule = 1; \ - wake_up_interruptible(&thp->waitq); \ - } while (0) -/** pr_debug_thread */ -#define pr_debug_thread(fmt, ...) -#endif - -/** - * @struct - xdma_kthread - * @brief xdma thread book keeping parameters - */ -struct xdma_kthread { - /** thread lock*/ - spinlock_t lock; - /** name of the thread */ - char name[16]; - /** cpu number for which the thread associated with */ - unsigned short cpu; - /** thread id */ - unsigned short id; - /** thread sleep timeout value */ - unsigned int timeout; - /** flags for thread */ - unsigned long flag; - /** thread wait queue */ - wait_queue_head_t waitq; - /* flag to indicate scheduling of thread */ - unsigned int schedule; - /** kernel task structure associated with thread*/ - struct task_struct *task; - /** thread work list count */ - unsigned int work_cnt; - /** thread work list count */ - struct list_head work_list; - /** thread initialization handler */ - int (*finit)(struct xdma_kthread *); - /** thread pending handler */ - int (*fpending)(struct list_head *); - /** thread peocessing handler */ - int (*fproc)(struct list_head *); - /** thread done handler */ - int (*fdone)(struct xdma_kthread *); -}; - - -/*****************************************************************************/ -/** - * xdma_threads_create() - create xdma threads -*********/ -int xdma_threads_create(unsigned int num_threads); - -/*****************************************************************************/ -/** - * xdma_threads_destroy() - destroy all the xdma threads created - * during system initialization - * - * @return none - *****************************************************************************/ -void xdma_threads_destroy(void); - -/*****************************************************************************/ -/** - * xdma_thread_remove_work() - handler to remove the attached work thread - * - * @param[in] engine: pointer to xdma_engine - * - * @return none - *****************************************************************************/ -void xdma_thread_remove_work(struct xdma_engine *engine); - -/*****************************************************************************/ -/** - * xdma_thread_add_work() - handler to add a work thread - * - * @param[in] engine: pointer to xdma_engine - * - * @return none - *****************************************************************************/ -void xdma_thread_add_work(struct xdma_engine *engine); - -#endif /* #ifndef __XDMA_KTHREAD_H__ */