sockmap: some performance optimizations

.github/actions/vmtest/action.yml

@@ -0,0 +1,39 @@
+name: 'vmtest'
+description: 'Build + run vmtest'
+inputs:
+  arch:
+    description: 'what arch to test'
+    required: true
+    default: 'x86_64'
+  toolchain:
+    description: 'what toolchain to use'
+    required: true
+    default: 'gcc'
+runs:
+  using: "composite"
+  steps:
+    # 1. Setup environment
+    - name: Setup build environment
+      uses: libbpf/ci/setup-build-env@master
+    # 2. Build
+    - name: Build kernel image
+      shell: bash
+      run: ${GITHUB_ACTION_PATH}/build.sh ${{ inputs.arch }} ${{ inputs.toolchain }}
+    - name: Build selftests
+      shell: bash
+      run: ${GITHUB_ACTION_PATH}/build_selftests.sh ${{ inputs.toolchain }}
+      env:
+        VMLINUX_BTF: ${{ github.workspace }}/vmlinux
+    # 3. Test
+    - name: Prepare rootfs
+      uses: libbpf/ci/prepare-rootfs@master
+      with:
+        project-name: 'libbpf'
+        arch: ${{ inputs.arch }}
+        kernel-root: '.'
+    - name: Run selftests
+      uses: libbpf/ci/run-qemu@master
+      with:
+        arch: ${{ inputs.arch}}
+        img: '/tmp/root.img'
+        vmlinuz: '${{ github.workspace }}/vmlinuz'

.github/actions/vmtest/build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ARCH="$1"
+TOOLCHAIN="$2"
+TOOLCHAIN_NAME="$(echo $TOOLCHAIN | cut -d '-' -f 1)"
+TOOLCHAIN_VERSION="$(echo $TOOLCHAIN | cut -d '-' -f 2)"
+
+if [ "$TOOLCHAIN_NAME" == "llvm" ]; then
+export LLVM="-$TOOLCHAIN_VERSION"
+fi
+
+THISDIR="$(cd $(dirname $0) && pwd)"
+
+source "${THISDIR}"/helpers.sh
+
+travis_fold start build_kernel "Building kernel with $TOOLCHAIN"
+
+cp ${GITHUB_WORKSPACE}/travis-ci/vmtest/configs/config-latest.${ARCH} .config
+
+make -j $((4*$(nproc))) olddefconfig all > /dev/null
+
+travis_fold end build_kernel

.github/actions/vmtest/build_selftests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euo pipefail
+
+THISDIR="$(cd $(dirname $0) && pwd)"
+
+source "${THISDIR}"/helpers.sh
+
+TOOLCHAIN="$1"
+TOOLCHAIN_NAME="$(echo $TOOLCHAIN | cut -d '-' -f 1)"
+TOOLCHAIN_VERSION="$(echo $TOOLCHAIN | cut -d '-' -f 2)"
+
+if [ "$TOOLCHAIN_NAME" == "llvm" ]; then
+export LLVM="-$TOOLCHAIN_VERSION"
+LLVM_VER=$TOOLCHAIN_VERSION
+else
+LLVM_VER=15
+fi
+
+travis_fold start prepare_selftests "Building selftests with $TOOLCHAIN"
+
+LIBBPF_PATH="${REPO_ROOT}"
+
+PREPARE_SELFTESTS_SCRIPT=${THISDIR}/prepare_selftests-${KERNEL}.sh
+if [ -f "${PREPARE_SELFTESTS_SCRIPT}" ]; then
+	(cd "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" && ${PREPARE_SELFTESTS_SCRIPT})
+fi
+
+if [[ "${KERNEL}" = 'LATEST' ]]; then
+	VMLINUX_H=
+else
+	VMLINUX_H=${THISDIR}/vmlinux.h
+fi
+
+cd ${REPO_ROOT}/${REPO_PATH}
+make \
+	CLANG=clang-${LLVM_VER} \
+	LLC=llc-${LLVM_VER} \
+	LLVM_STRIP=llvm-strip-${LLVM_VER} \
+	VMLINUX_BTF="${VMLINUX_BTF}" \
+	VMLINUX_H="${VMLINUX_H}" \
+	-C "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \
+	-j $((4*$(nproc))) > /dev/null
+cd -
+mkdir "${LIBBPF_PATH}"/selftests
+cp -R "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \
+	"${LIBBPF_PATH}"/selftests
+cd "${LIBBPF_PATH}"
+rm selftests/bpf/.gitignore
+git add selftests
+
+travis_fold end prepare_selftests

.github/actions/vmtest/helpers.sh

@@ -0,0 +1,44 @@
+# $1 - start or end
+# $2 - fold identifier, no spaces
+# $3 - fold section description
+travis_fold() {
+  local YELLOW='\033[1;33m'
+  local NOCOLOR='\033[0m'
+  if [ -z ${GITHUB_WORKFLOW+x} ]; then
+    echo travis_fold:$1:$2
+    if [ ! -z "${3:-}" ]; then
+      echo -e "${YELLOW}$3${NOCOLOR}"
+    fi
+    echo
+  else
+    if [ $1 = "start" ]; then
+      line="::group::$2"
+      if [ ! -z "${3:-}" ]; then
+        line="$line - ${YELLOW}$3${NOCOLOR}"
+      fi
+    else
+      line="::endgroup::"
+    fi
+    echo -e "$line"
+  fi
+}
+
+__print() {
+  local TITLE=""
+  if [[ -n $2 ]]; then
+      TITLE=" title=$2"
+  fi
+  echo "::$1${TITLE}::$3"
+}
+
+# $1 - title
+# $2 - message
+print_error() {
+  __print error $1 $2
+}
+
+# $1 - title
+# $2 - message
+print_notice() {
+  __print notice $1 $2
+}

.github/workflows/test.yml

@@ -0,0 +1,57 @@
+name: bpf-ci
+
+on:
+  pull_request:
+
+concurrency: 
+  group: ci-test-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  VM_Test:
+    runs-on: ${{ matrix.runs_on }}
+    name: Kernel ${{ matrix.kernel }} on ${{ matrix.runs_on }} with ${{ matrix.toolchain }}
+    timeout-minutes: 100
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - kernel: 'LATEST'
+            runs_on: ubuntu-latest
+            arch: 'x86_64'
+            toolchain: 'gcc'
+          - kernel: 'LATEST'
+            runs_on: ubuntu-latest
+            arch: 'x86_64'
+            toolchain: 'llvm-15'
+          - kernel: 'LATEST'
+            runs_on: z15
+            arch: 's390x'
+            toolchain: 'gcc'
+    env:
+      AUTHOR_EMAIL: "$(git log -1 --pretty=\"%aE\")"
+      KERNEL: LATEST
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+    steps:
+      - uses: actions/checkout@v2
+      - if: ${{ github.repository != 'kernel-patches/bpf' && github.repository != 'kernel-patches/bpf-rc' }}
+        name: Download bpf-next tree
+        uses: libbpf/ci/get-linux-source@master
+        with:
+          dest: '.kernel'
+      - if: ${{ github.repository != 'kernel-patches/bpf' && github.repository != 'kernel-patches/bpf-rc' }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          rm -rf .kernel/.git
+          cp -rf .kernel/. .
+          rm -rf .kernel
+      - uses: libbpf/ci/patch-kernel@master
+        with:
+          patches-root: '${{ github.workspace }}/travis-ci/diffs'
+          repo-root: '${{ github.workspace }}'
+      - uses: ./.github/actions/vmtest
+        with:
+          arch: ${{ matrix.arch }}
+          toolchain: ${{ matrix.toolchain }}

README

@@ -1,18 +0,0 @@
-Linux kernel
-============
-
-There are several guides for kernel developers and users. These guides can
-be rendered in a number of formats, like HTML and PDF. Please read
-Documentation/admin-guide/README.rst first.
-
-In order to build the documentation, use ``make htmldocs`` or
-``make pdfdocs``.  The formatted documentation can also be read online at:
-
-    https://www.kernel.org/doc/html/latest/
-
-There are various text files in the Documentation/ subdirectory,
-several of them using the Restructured Text markup notation.
-
-Please read the Documentation/process/changes.rst file, as it contains the
-requirements for building and running the kernel, and information about
-the problems which may result by upgrading your kernel.

include/linux/net.h

@@ -152,6 +152,8 @@ struct module;
 struct sk_buff;
 typedef int (*sk_read_actor_t)(read_descriptor_t *, struct sk_buff *,
 			       unsigned int, size_t);
+typedef int (*skb_read_actor_t)(struct sock *, struct sk_buff *);
+
 
 struct proto_ops {
 	int		family;
@@ -214,6 +216,8 @@ struct proto_ops {
 	 */
 	int		(*read_sock)(struct sock *sk, read_descriptor_t *desc,
 				     sk_read_actor_t recv_actor);
+	/* This is different from read_sock(), it reads an entire skb at a time. */
+	int		(*read_skb)(struct sock *sk, skb_read_actor_t recv_actor);
 	int		(*sendpage_locked)(struct sock *sk, struct page *page,
 					   int offset, size_t size, int flags);
 	int		(*sendmsg_locked)(struct sock *sk, struct msghdr *msg,

include/net/tcp.h

@@ -672,6 +672,7 @@ void tcp_get_info(struct sock *, struct tcp_info *);
 /* Read 'sendfile()'-style from a TCP socket */
 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
 		  sk_read_actor_t recv_actor);
+int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
 
 void tcp_initialize_rcv_mss(struct sock *sk);
 

include/net/udp.h

@@ -306,8 +306,7 @@ struct sock *__udp6_lib_lookup(struct net *net,
 			       struct sk_buff *skb);
 struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb,
 				 __be16 sport, __be16 dport);
-int udp_read_sock(struct sock *sk, read_descriptor_t *desc,
-		  sk_read_actor_t recv_actor);
+int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
 
 /* UDP uses skb->dev_scratch to cache as much information as possible and avoid
  * possibly multiple cache miss on dequeue()

net/core/skmsg.c

@@ -497,23 +497,27 @@ bool sk_msg_is_readable(struct sock *sk)
 }
 EXPORT_SYMBOL_GPL(sk_msg_is_readable);
 
-static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
-						  struct sk_buff *skb)
+static struct sk_msg *alloc_sk_msg(void)
 {
 	struct sk_msg *msg;
 
-	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
+	msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_KERNEL);
+	if (unlikely(!msg))
 		return NULL;
+	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
+	return msg;
+}
 
-	if (!sk_rmem_schedule(sk, skb, skb->truesize))
+static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
+						  struct sk_buff *skb)
+{
+	if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
 		return NULL;
 
-	msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_KERNEL);
-	if (unlikely(!msg))
+	if (!sk_rmem_schedule(sk, skb, skb->truesize))
 		return NULL;
 
-	sk_msg_init(msg);
-	return msg;
+	return alloc_sk_msg();
 }
 
 static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
@@ -590,13 +594,12 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
 				     u32 off, u32 len)
 {
-	struct sk_msg *msg = kzalloc(sizeof(*msg), __GFP_NOWARN | GFP_ATOMIC);
+	struct sk_msg *msg = alloc_sk_msg();
 	struct sock *sk = psock->sk;
 	int err;
 
 	if (unlikely(!msg))
 		return -EAGAIN;
-	sk_msg_init(msg);
 	skb_set_owner_r(skb, sk);
 	err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
 	if (err < 0)
@@ -1160,21 +1163,14 @@ static void sk_psock_done_strp(struct sk_psock *psock)
 }
 #endif /* CONFIG_BPF_STREAM_PARSER */
 
-static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
-				 unsigned int offset, size_t orig_len)
+static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
 {
-	struct sock *sk = (struct sock *)desc->arg.data;
 	struct sk_psock *psock;
 	struct bpf_prog *prog;
 	int ret = __SK_DROP;
-	int len = orig_len;
+	int len = skb->len;
 
-	/* clone here so sk_eat_skb() in tcp_read_sock does not drop our data */
-	skb = skb_clone(skb, GFP_ATOMIC);
-	if (!skb) {
-		desc->error = -ENOMEM;
-		return 0;
-	}
+	skb_get(skb);
 
 	rcu_read_lock();
 	psock = sk_psock(sk);
@@ -1187,12 +1183,10 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
 	if (!prog)
 		prog = READ_ONCE(psock->progs.skb_verdict);
 	if (likely(prog)) {
-		skb->sk = sk;
 		skb_dst_drop(skb);
 		skb_bpf_redirect_clear(skb);
 		ret = bpf_prog_run_pin_on_cpu(prog, skb);
 		ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
-		skb->sk = NULL;
 	}
 	if (sk_psock_verdict_apply(psock, skb, ret) < 0)
 		len = 0;
@@ -1204,16 +1198,10 @@ static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
 static void sk_psock_verdict_data_ready(struct sock *sk)
 {
 	struct socket *sock = sk->sk_socket;
-	read_descriptor_t desc;
 
-	if (unlikely(!sock || !sock->ops || !sock->ops->read_sock))
+	if (unlikely(!sock || !sock->ops || !sock->ops->read_skb))
 		return;
-
-	desc.arg.data = sk;
-	desc.error = 0;
-	desc.count = 1;
-
-	sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv);
+	sock->ops->read_skb(sk, sk_psock_verdict_recv);
 }
 
 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)