Merge pull request #64 from lec-bit/image-online-L7

support compile online image

Author: nlgwcy

bpf/kmesh/include/kmesh_common.h

@@ -50,12 +50,12 @@
 		val;								   \
 	})
 
-#if !ENHANCED_KERNEL
 struct bpf_mem_ptr {
 	void *ptr;
 	__u32 size;
 };
 
+#if !ENHANCED_KERNEL
 static inline int bpf__strncmp (char *dst, int n, const char *src) {
 	if (dst == NULL || src == NULL)
 		return -1;

build.sh

@@ -13,7 +13,20 @@ function bpf_compile_range_adjust() {
 }
 
 function set_enhanced_kernel_env() {
-    if grep -q "FN(parse_header_msg)" /usr/include/linux/bpf.h; then
+    # we use /usr/include/linux/bpf.h to determine the runtime environment’s 
+    # support for kmesh. Considering the case of online image compilation, a 
+    # variable KERNEL_HEADER_LINUX_BPF is used here to specify the path of the
+    # source of macro definition. 
+    # When using an online compiled image, /usr/include/linux/bpf.h in host 
+    # machine  will be mounted to config/linux-bpf.h. 
+    # Otherwise, /usr/include/linux/bpf.h from the current compilation 
+    # environment will be obtained
+    export KERNEL_HEADER_LINUX_BPF=$ROOT_DIR/config/linux-bpf.h
+    if [ ! -f "$KERNEL_HEADER_LINUX_BPF" ]; then
+	    export KERNEL_HEADER_LINUX_BPF=/usr/include/linux/bpf.h
+    fi
+
+    if grep -q "FN(parse_header_msg)" $KERNEL_HEADER_LINUX_BPF; then
 	    export ENHANCED_KERNEL="enhanced"
     else
 	    export ENHANCED_KERNEL="unenhanced"

build/docker/kmesh.dockerfile

@@ -3,11 +3,11 @@
 # 
 # Usage:
 # docker build -f kmesh.dockerfile -t kmesh:latest .
-# docker run -itd --privileged=true -v /etc/cni/net.d:/etc/cni/net.d -v /opt/cni/bin:/opt/cni/bin -v /mnt:/mnt -v /sys/fs/bpf:/sys/fs/bpf -v /lib/modules:/lib/modules --name kmesh kmesh:latest
+# docker run -itd --privileged=true -v /usr/src:/usr/src -v /usr/include/linux/bpf.h:/kmesh/config/linux-bpf.h -v /etc/cni/net.d:/etc/cni/net.d -v /opt/cni/bin:/opt/cni/bin -v /mnt:/mnt -v /sys/fs/bpf:/sys/fs/bpf -v /lib/modules:/lib/modules --name kmesh kmesh:latest
 #
 
 # base image
-FROM openeuler/openeuler:23.03
+FROM openeuler/openeuler:23.09
 
 # container work directory
 WORKDIR /kmesh
@@ -17,12 +17,11 @@ WORKDIR /kmesh
 ADD . /kmesh
 
 # install pkg dependencies 
-# RUN yum install -y kmod util-linux kmesh
+# RUN yum install -y kmod util-linux
+# install package in online-compile image
 RUN yum install -y kmod \
     && yum install -y util-linux \
-    && yum install -y kmesh-*.rpm \
-    && yum clean all \
-    && rm -rf /var/cache/yum
+    && yum install -y make golang clang llvm libboundscheck protobuf-c-devel bpftool libbpf libbpf-devel cmake
 
 RUN chmod +x start_kmesh.sh
 

build/docker/kmesh.yaml

@@ -34,6 +34,17 @@ spec:
          - name: kmesh-cniplugin-install-path
            hostPath:
              path: /opt/cni/bin
+         # Optional: 
+         # online compilation image needs to determine the support 
+         # to kmesh in host during compilation, based on this file.
+         - name: linux-bpf
+           hostPath:
+             path: /usr/include/linux/bpf.h
+         # Optional: 
+         # online compilation image needs compile kmesh.ko by host file
+         - name: ko-build-path
+           hostPath:
+             path: /usr/src
        containers:
          - name: kmesh
            image: kmesh:latest
@@ -65,13 +76,27 @@ spec:
            - name: kube-config-path
              mountPath: /root/.kube
              readOnly: true
+           # k8s default cni conflist path
            - name: cni
              mountPath: /etc/cni/net.d
              readOnly: false
+           # k8s deafult cni path
            - name: kmesh-cniplugin-install-path
              mountPath: /opt/cni/bin
              readOnly: false
+           # Optional: 
+           # online compilation image needs to determine the support 
+           # to kmesh in host during compilation, based on this file.           
+           - name: linux-bpf
+             mountPath: /kmesh/config/linux-bpf.h
+             readOnly: true
+           # Optional: 
+           # online compilation image needs compile kmesh.ko by host file
+           - name: ko-build-path
+             mountPath: /usr/src
+             readOnly: true
            resources:
              limits:
-               memory: "200Mi"
+               # image online-compile needs 800Mi, or only 200Mi
+               memory: "800Mi"
                cpu: "1"

kmesh_bpf_env.sh

@@ -15,9 +15,9 @@ helper_name=(
 	get_msg_header_element
 )
 
-base_line=`grep -nr "FN(unspec)" /usr/include/linux/bpf.h | awk -F ":" {'print $1'}`
+base_line=`grep -nr "FN(unspec)" $KERNEL_HEADER_LINUX_BPF | awk -F ":" {'print $1'}`
 for name in ${helper_name[@]}; do
-	current_line=`grep -nr "FN($name)" /usr/include/linux/bpf.h | awk -F ":" {'print $1'}`
+	current_line=`grep -nr "FN($name)" $KERNEL_HEADER_LINUX_BPF | awk -F ":" {'print $1'}`
 	if [ -n "$current_line" ]; then
 		helper_id=`expr $current_line - $base_line`
 		sed -Ei "/$name/s/([0-9]+)[^0-9]*$/$helper_id;/" $ROOT_DIR/depends/include/bpf_helper_defs_ext.h

kmesh_compile_env_pre.sh

@@ -14,11 +14,11 @@ function install_libboundscheck() {
 function dependency_pkg_install() {
     if command -v apt > /dev/null; then
 	    # apt install 
-	    apt-get update && apt-get install -y git make clang libbpf-dev llvm rpm linux-tools-generic protobuf-compiler libprotobuf-dev libprotobuf-c-dev protobuf-c-compiler cmake golang
+	    apt-get update && apt-get install -y git make clang libbpf-dev llvm linux-tools-generic protobuf-compiler libprotobuf-dev libprotobuf-c-dev protobuf-c-compiler cmake golang
 	    install_libboundscheck
     elif command -v yum > /dev/null; then
 	    # yum install
-	    yum install -y git make golang clang llvm libboundscheck protobuf protobuf-c protobuf-c-devel bpftool rpm-build rpmdevtools libbpf libbpf-devel cmake
+	    yum install -y git make golang clang llvm libboundscheck protobuf protobuf-c protobuf-c-devel bpftool libbpf libbpf-devel cmake
     fi
 }
 
@@ -41,6 +41,18 @@ function adapt_low_version_kernel() {
     fi
 }
 
+# Special case: 
+# There is a structure that is only defined in certain environments and is 
+# only used during the compilation stage. Therefore, the definition of this 
+# structure in the include directory is dynamically adjusted according to 
+# the current compilation environment during compilation.
+function adapt_include_env {
+    if grep -q "struct bpf_mem_ptr {" /usr/include/linux/bpf.h; then
+        sed -i '/bpf_mem_ptr/{N;N;N;N;d;}' bpf/kmesh/include/kmesh_common.h
+    fi
+}
+
 dependency_pkg_install
 fix_libbpf_bug
 adapt_low_version_kernel
+adapt_include_env

kmesh_macros_env.sh

@@ -7,21 +7,21 @@ function set_config() {
 }
 
 # MDA_LOOPBACK_ADDR
-if grep -q "FN(get_netns_cookie)" /usr/include/linux/bpf.h; then
+if grep -q "FN(get_netns_cookie)" $KERNEL_HEADER_LINUX_BPF; then
 	set_config MDA_LOOPBACK_ADDR 1
 else
 	set_config MDA_LOOPBACK_ADDR 0
 fi
 
 # MDA_NAT_ACCEL
-if grep -q "FN(sk_original_addr)" /usr/include/linux/bpf.h; then
+if grep -q "FN(sk_original_addr)" $KERNEL_HEADER_LINUX_BPF; then
 	set_config MDA_NAT_ACCEL 1
 else
 	set_config MDA_NAT_ACCEL 0
 fi
 
 # MDA_GID_UID_FILTER
-if grep -q "FN(get_sockops_uid_gid)" /usr/include/linux/bpf.h; then
+if grep -q "FN(get_sockops_uid_gid)" $KERNEL_HEADER_LINUX_BPF; then
 	set_config MDA_GID_UID_FILTER 1
 else
 	set_config MDA_GID_UID_FILTER 0
@@ -42,7 +42,7 @@ else
 fi
 
 # ENHANCED_KERNEL
-if grep -q "FN(parse_header_msg)" /usr/include/linux/bpf.h; then
+if grep -q "FN(parse_header_msg)" $KERNEL_HEADER_LINUX_BPF; then
 	set_config ENHANCED_KERNEL 1
 else
 	set_config ENHANCED_KERNEL 0