Merge branch 'master' into dev/add_energy_scheduler

.github/workflows/build.yaml

@@ -1,4 +1,4 @@
-name: Build and Upload Packages
+name: Build, Upload and Test
 
 on:
   push:
@@ -9,95 +9,152 @@ on:
       - master
       - release/*
   workflow_dispatch:
+    inputs:
+      frontend_branch:
+        description: 'Branch of FrontEnd to use'
+        required: true
+        default: 'master'
+
+env:
+  SCRIPT_PATH: /opt/actions-runner-external/script
+  CACHE_PATH: /opt/actions-runner-external/cache
+  TEST_FAILED: false
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/nativu5/cranedev:ci
-      credentials:
-         username: ${{ github.actor }}
-         password: ${{ github.token }}
+    if: github.event.pull_request.draft == false
+    runs-on: self-hosted
     defaults:
       run:
         shell: bash -leo pipefail {0}
 
     steps:
+      # Checkout repo
       - name: Checkout code
         uses: actions/checkout@v4
+        with: 
+          path: CraneSched
 
-      - name: Configure
-        run: |
-          echo "Configuring at $(pwd)"
-          mkdir -p out
-          cmake -G Ninja \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DCRANE_FULL_DYNAMIC=OFF \
-            -S . -B out
-
-      - name: Build
+      # Checkout frontend
+      - name: Checkout frontend
+        uses: actions/checkout@v4
+        with:
+          repository: PKUHPC/CraneSched-FrontEnd
+          path: CraneSched-FrontEnd
+          ref: ${{ inputs.frontend_branch }}
+
+      # Pull CI Image and Prepare
+      - name: Pull CI Image
         run: |
-          echo "Building at $(pwd)"
-          cmake --build out --config Release
+          podman pull ghcr.io/nativu5/cranedev:ci
+          echo "Linking $SCRIPT_PATH to $(pwd)/script"
+          ln -sfT $SCRIPT_PATH $(pwd)/script
+          echo "Linking $CACHE_PATH to $(pwd)/cache"
+          ln -sfT $CACHE_PATH $(pwd)/cache
+          mkdir -p output
+          mkdir -p log
 
-      - name: Package with CPack
+      - name: Build in Container
         run: |
-          echo "Packaging with CPack"
-          pushd out
-          cpack
-          popd
+          podman run --rm \
+              -v ./CraneSched:/Workspace/CraneSched \
+              -v ./CraneSched-FrontEnd:/Workspace/CraneSched-FrontEnd \
+              -v ./output:/Workspace/output \
+              -v ./script:/Workspace/script \
+              -v ./cache/ccache:/root/.ccache \
+              ghcr.io/nativu5/cranedev:ci /bin/bash --login script/build.sh
 
-      - name: Collect packages
+      # Collect and upload artifacts
+      - name: Count Artifacts
         run: |
-          mkdir -p upload/packages
-          cp out/*.deb upload/packages/ || true
-          cp out/*.rpm upload/packages/ || true
-          for package in upload/packages/*; do
-            package_name=$(basename "$package")
-            case "$package_name" in
-              *craned.rpm)
-                echo "ARTIFACT_NAME_CRANED_RPM=$package_name" >> $GITHUB_ENV
-                ;;
-              *craned.deb)
-                echo "ARTIFACT_NAME_CRANED_DEB=$package_name" >> $GITHUB_ENV
-                ;;
-              *cranectld.rpm)
-                echo "ARTIFACT_NAME_CRANECTLD_RPM=$package_name" >> $GITHUB_ENV
-                ;;
-              *cranectld.deb)
-                echo "ARTIFACT_NAME_CRANECTLD_DEB=$package_name" >> $GITHUB_ENV
-                ;;
-            esac
-          done
+          ARTIFACT_NAME_CRANED_DEB=$(basename $(ls output/*craned.deb))
+          ARTIFACT_NAME_CRANED_RPM=$(basename $(ls output/*craned.rpm))
+          ARTIFACT_NAME_CRANECTLD_DEB=$(basename $(ls output/*cranectld.deb))
+          ARTIFACT_NAME_CRANECTLD_RPM=$(basename $(ls output/*cranectld.rpm))
+          ARTIFACT_NAME_FRONTEND=bin
+          ARTIFACT_NAME_PLUGIN=plugin
+          echo "ARTIFACT_NAME_CRANED_DEB=$ARTIFACT_NAME_CRANED_DEB" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME_CRANED_RPM=$ARTIFACT_NAME_CRANED_RPM" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME_CRANECTLD_DEB=$ARTIFACT_NAME_CRANECTLD_DEB" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME_CRANECTLD_RPM=$ARTIFACT_NAME_CRANECTLD_RPM" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME_FRONTEND=$ARTIFACT_NAME_FRONTEND" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME_PLUGIN=$ARTIFACT_NAME_PLUGIN" >> $GITHUB_ENV
 
       - name: Upload craned .deb package
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME_CRANED_DEB }}
-          path: upload/packages/${{ env.ARTIFACT_NAME_CRANED_DEB }}
-          retention-days: 30
+          path: output/${{ env.ARTIFACT_NAME_CRANED_DEB }}
+          retention-days: 14
           if-no-files-found: error
 
       - name: Upload craned .rpm package
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME_CRANED_RPM }}
-          path: upload/packages/${{ env.ARTIFACT_NAME_CRANED_RPM }}
-          retention-days: 30
+          path: output/${{ env.ARTIFACT_NAME_CRANED_RPM }}
+          retention-days: 14
           if-no-files-found: error
 
       - name: Upload cranectld .deb package
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME_CRANECTLD_DEB }}
-          path: upload/packages/${{ env.ARTIFACT_NAME_CRANECTLD_DEB }}
-          retention-days: 30
+          path: output/${{ env.ARTIFACT_NAME_CRANECTLD_DEB }}
+          retention-days: 14
           if-no-files-found: error
 
       - name: Upload cranectld .rpm package
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.ARTIFACT_NAME_CRANECTLD_RPM }}
-          path: upload/packages/${{ env.ARTIFACT_NAME_CRANECTLD_RPM }}
-          retention-days: 30
+          path: output/${{ env.ARTIFACT_NAME_CRANECTLD_RPM }}
+          retention-days: 14
+          if-no-files-found: error
+
+      - name: Upload frontend
+        uses: actions/upload-artifact@v4
+        with:
+          name: frontend
+          path: output/${{ env.ARTIFACT_NAME_FRONTEND }}
+          retention-days: 14
           if-no-files-found: error
+
+      - name: Upload plugin
+        uses: actions/upload-artifact@v4
+        with:
+          name: plugin
+          path: output/${{ env.ARTIFACT_NAME_PLUGIN }}
+          retention-days: 14
+          if-no-files-found: error
+
+      # Run AutoTest
+      - name: Run AutoTest
+        continue-on-error: true
+        run: |
+          NETWORK_ID=$(podman container inspect mongodb | jq -r '.[0].NetworkSettings.Networks | keys[]')
+          podman run -d --rm --name autotest \
+            --privileged \
+            --systemd true \
+            -v /lib/modules:/lib/modules:ro \
+            -v ./script:/CraneSched-AutoTest/script \
+            -v ./output:/CraneSched-AutoTest/output \
+            -v ./log:/CraneSched-AutoTest/log \
+            --network $NETWORK_ID \
+            localhost/autotest
+          podman exec autotest /bin/bash --login script/run.sh || echo "TEST_FAILED=true" >> $GITHUB_ENV
+          podman stop autotest
+      
+      # Upload AutoTest Results
+      - name: Upload AutoTest Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: result.json
+          path: output/result.json
+          retention-days: 14
+          if-no-files-found: error
+
+      # Alarm if has failed cases
+      - name: Alarm on test failure
+        if: env.TEST_FAILED == 'true'
+        run: exit 1

CMakeLists.txt

@@ -43,9 +43,18 @@ if (VERSION_CONTENT STREQUAL "")
     set(CMAKE_PROJECT_VERSION "0.0.0")
 endif ()
 
+# If ccache is found, set it as the compiler launcher
+find_program(CCACHE_PROGRAM ccache)
+if (CCACHE_PROGRAM)
+    message(STATUS "ccache found. Use ccache to launch compilers.")
+    set_property(GLOBAL PROPERTY CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
+    set_property(GLOBAL PROPERTY CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
+endif ()
+
 project(Crane VERSION ${CMAKE_PROJECT_VERSION} LANGUAGES C CXX)
-# check and set compiler
-set(REQUIRED_GNU_VERSION 13.0.0)
+
+# Check and set compiler
+set(REQUIRED_GNU_VERSION 14.0.0)
 set(REQUIRED_CLANG_VERSION 19.0.0)
 set(REQUIRED_BPF_CLANG_VERSION 17.0.0)
 
@@ -252,13 +261,6 @@ else ()
     message(STATUS "IPO / LTO not supported: <${error}>")
 endif ()
 
-find_program(CCACHE_PROGRAM ccache)
-if (CCACHE_PROGRAM)
-    message(STATUS "ccache found. Use ccache to launch compilers.")
-    set_property(GLOBAL PROPERTY CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
-    set_property(GLOBAL PROPERTY CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
-endif ()
-
 if (${CRANE_FULL_DYNAMIC})
     set(CMAKE_SKIP_BUILD_RPATH FALSE)
     set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
@@ -278,6 +280,7 @@ set(DEPENDENCIES_PRE_INSTALLED_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/pre_
 add_subdirectory(${DEPENDENCIES_PRE_INSTALLED_DIR})
 
 find_package(Threads REQUIRED)
+find_library(LIBUTIL_LIBRARY util)
 
 # New in version cmake3.24:
 # Set ZLIB_USE_STATIC_LIBS to ON to look for static libraries. Default is OFF.
@@ -394,6 +397,14 @@ install(TARGETS craned
         PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_WRITE GROUP_EXECUTE WORLD_READ WORLD_WRITE WORLD_EXECUTE
 )
 
+if(ENABLE_BPF)
+    install(FILES 
+            ${CMAKE_BINARY_DIR}/src/Misc/BPF/cgroup_dev_bpf.o
+            DESTINATION /usr/local/lib64/bpf/
+            COMPONENT cranedc
+            PERMISSIONS OWNER_READ GROUP_READ WORLD_READ)
+endif()
+
 # Install unit files
 install(FILES
         ${CMAKE_BINARY_DIR}/etc/cranectld.service
@@ -439,16 +450,19 @@ set(CPACK_PACKAGE_DESCRIPTION "An HPC and Cloud Computing Fused Job Scheduling S
 set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE")
 set(CPACK_RESOURCE_FILE_README "${CMAKE_SOURCE_DIR}/README.md")
 set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CMAKE_PROJECT_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
+
 # Do not allow the package to be relocated
 set(CPACK_PACKAGE_RELOCATABLE OFF)
 
 # RPM
 set(CPACK_RPM_PACKAGE_GROUP "System Environment/Base")
 set(CPACK_RPM_PACKAGE_LICENSE "AGPL v3")
 set(CPACK_RPM_COMPRESSION_TYPE "xz")
-#TODO CPACK_PACKAGE_DESCRIPTION
+
 # DEB
 set(CPACK_DEB_COMPONENT_INSTALL ON)
 set(CPACK_DEBIAN_COMPRESSION_TYPE "xz")
+# Seperate the debuginfo to avoid large DEB package
+set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ON)
 
 include(CPack)

dependencies/cmake/CMakeLists.txt

@@ -18,7 +18,6 @@ add_subdirectory(yaml-cpp)
 add_subdirectory(fmt)
 add_subdirectory(googletest)
 add_subdirectory(spdlog)
-#add_subdirectory(LibEvent)
 add_subdirectory(cxxopts)
 add_subdirectory(grpc)
 add_subdirectory(libcgroup)
@@ -29,7 +28,7 @@ add_subdirectory(ranges-v3)
 add_subdirectory(backward-cpp)
 add_subdirectory(fpm)
 add_subdirectory(fast-cpp-csv-parser)
-#add_subdirectory(mariadb-connector-c)
+add_subdirectory(curl)
 
 include(${CMAKE_SOURCE_DIR}/CMakeModule/SuppressHeaderWarning.cmake)
 suppress_header_warning()

dependencies/cmake/curl/CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_policy(SET CMP0069 NEW)
+set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
+
+include(FetchContent)
+
+if (CRANE_USE_GITEE_SOURCE)
+    # TODO: replace with gitee source
+else ()
+    set(CURL_URL "https://github.com/curl/curl/releases/download/curl-8_5_0/curl-8.5.0.tar.gz")
+endif ()
+
+FetchContent_Declare(
+    curl
+    URL ${CURL_URL}
+    URL_HASH SHA256=05fc17ff25b793a437a0906e0484b82172a9f4de02be5ed447e0cab8c3475add
+)
+
+FetchContent_GetProperties(curl)
+if(NOT curl_POPULATED)
+    set(BUILD_SHARED_LIBS OFF)
+    set(BUILD_TESTING OFF)
+    set(HTTP_ONLY ON)
+    set(ENABLE_MANUAL OFF)
+    set(BUILD_CURL_EXE OFF)
+    set(CURL_DISABLE_LDAP ON)
+    set(CURL_DISABLE_LDAPS ON)
+    set(CURL_DISABLE_TELNET ON)
+    set(CURL_DISABLE_DICT ON)
+    set(CURL_DISABLE_FILE ON)
+    set(CURL_DISABLE_TFTP ON)
+    set(CURL_DISABLE_RTSP ON)
+    set(CURL_DISABLE_POP3 ON)
+    set(CURL_DISABLE_IMAP ON)
+    set(CURL_DISABLE_SMTP ON)
+    set(CURL_DISABLE_GOPHER ON)
+
+    FetchContent_Populate(curl)
+    add_subdirectory(${curl_SOURCE_DIR} ${curl_BINARY_DIR})
+
+    # add_library(CURL::libcurl ALIAS libcurl)
+endif()