Ref #3023: Add native compilation with sources

.github/actions/e2e-install-native/action.yml

@@ -25,6 +25,10 @@ inputs:
   cluster-kube-config-data:
     description: 'Base16 encoded kube config - required for custom cluster type only'
     required: false
+  high-memory:
+    description: 'Indicates whether the high memory integration tests should be launched (default false)'
+    default: 'false'
+    required: false
 
 runs:
   using: "composite"
@@ -90,7 +94,7 @@ runs:
     name: List Tests Marked As Problematic
     uses: ./.github/actions/kamel-report-problematic
     with:
-      test-suite: namespace/install
+      test-suite: namespace/native
 
   - id: run-it
     name: Run IT
@@ -104,6 +108,8 @@ runs:
         -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \
         -s "${{steps.config-cluster.outputs.cluster-image-registry-insecure }}" \
         -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \
+        -t "${{ inputs.high-memory }}" \
+        -q "${{ env.CAMEL_K_LOG_LEVEL }}" \
         -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}"
 
   - name: Cleanup

.github/actions/e2e-install-native/exec-tests.sh

@@ -25,7 +25,7 @@
 
 set -e
 
-while getopts ":b:c:i:l:n:s:v:x:" opt; do
+while getopts ":b:c:i:l:n:q:s:t:v:x:" opt; do
   case "${opt}" in
     b)
       BUILD_CATALOG_SOURCE_NAME=${OPTARG}
@@ -42,9 +42,15 @@ while getopts ":b:c:i:l:n:s:v:x:" opt; do
     n)
       IMAGE_NAME=${OPTARG}
       ;;
+    q)
+      LOG_LEVEL=${OPTARG}
+      ;;
     s)
       REGISTRY_INSECURE=${OPTARG}
       ;;
+    t)
+      HIGH_MEMORY=${OPTARG}
+      ;;
     v)
       IMAGE_VERSION=${OPTARG}
       ;;
@@ -116,5 +122,14 @@ export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE}
 export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION}
 export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS}
 
+export CAMEL_K_TEST_LOG_LEVEL="${LOG_LEVEL}"
+if [ "${LOG_LEVEL}" == "debug" ]; then
+  export CAMEL_K_TEST_MAVEN_CLI_OPTIONS="-X ${CAMEL_K_TEST_MAVEN_CLI_OPTIONS}"
+fi
+
 # Then run integration tests
-DO_TEST_PREBUILD=false make test-quarkus-native
+if [ "${HIGH_MEMORY}" == "true" ]; then
+  DO_TEST_PREBUILD=false make test-quarkus-native-high-memory
+else
+  DO_TEST_PREBUILD=false make test-quarkus-native
+fi

.github/actions/kamel-build-binary/build-binary.sh

@@ -97,7 +97,11 @@ fi
 
 echo "Moving kamel binary to be visible on PATH"
 
-${SUDO} mv ./kamel /usr/bin
+if [ "$RUNNER_OS" == "macOS" ]; then
+  ${SUDO} mv ./kamel /usr/local/bin
+else
+  ${SUDO} mv ./kamel /usr/bin
+fi
 echo "Kamel version installed: $(kamel version)"
 
 #

.github/actions/kamel-config-cluster-kind/action.yml

@@ -23,11 +23,12 @@ runs:
   steps:
     - id: install-cluster
       name: Install Cluster
-      uses: container-tools/kind-action@v1
+      uses: container-tools/kind-action@61f1afd4807b0dac84f3232ec99e45c63701d220
       if: ${{ env.CLUSTER_KIND_CONFIGURED != 'true' }}
       with:
         version: v0.14.0
         node_image: kindest/node:v1.23.6@sha256:b1fa224cc6c7ff32455e0b1fd9cbfd3d3bc87ecaa8fcb06961ed1afb3db0f9ae
+        cpu: 3
 
     - id: info
       name: Info

.github/actions/kamel-preflight-test/preflight-test.sh

@@ -38,7 +38,7 @@ waitForOperator() {
   #
   # Wait for the operator to be running
   #
-  local timeout=180
+  local timeout=360
   local i=1
   local command="kubectl get pods -n ${NAMESPACE} 2> /dev/null | grep camel-k | grep Running &> /dev/null"
 

.github/actions/kamel-prepare-env/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: Set Env Vars
       shell: bash
       run: |
-        if [ "$RUNNER_OS" == "Linux" ]; then
+        if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
           SUDO=sudo
         fi
 
@@ -54,10 +54,11 @@ runs:
           ${{ env.SUDO }} rm -rf "/usr/local/share/boost"
           ${{ env.SUDO }} rm -rf "$AGENT_TOOLSDIRECTORY"
         fi
-
-        df -kh
-        docker rmi $(docker image ls -aq) || true # Don't fail if image is not present
-        df -kh
+        if [ -x "$(command -v docker)" ]; then
+          df -kh
+          docker rmi $(docker image ls -aq) || true # Don't fail if image is not present
+          df -kh
+        fi
 
         echo "Final status:"
         df -h
@@ -101,6 +102,15 @@ runs:
       with:
         version: 'latest'
 
+    - id: install-realpath
+      shell: bash
+      if: ${{ env.KAMEL_PREPARE_ENV != 'true' }}
+      run : |
+        if [ "$RUNNER_OS" == "macOS" ] && ! [ -x "$(command -v realpath)" ]; then
+          echo 'Installing realpath...'
+          brew install coreutils
+        fi
+
     - id: complete-action
       name: Environment Prepared
       shell: bash

.github/workflows/native.yml

@@ -0,0 +1,118 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: native
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+  pull_request:
+    branches:
+      - main
+      - "release-*"
+    paths-ignore:
+      - 'docs/**'
+      - 'proposals/**'
+      - '**.adoc'
+      - '**.md'
+      - 'KEYS'
+      - 'LICENSE'
+      - 'NOTICE'
+  push:
+    branches:
+      - main
+      - "release-*"
+    paths-ignore:
+      - 'docs/**'
+      - 'proposals/**'
+      - '**.adoc'
+      - '**.md'
+      - 'KEYS'
+      - 'LICENSE'
+      - 'NOTICE'
+  workflow_dispatch:
+    inputs:
+      log-level:
+        description: 'Set the operator log level (info or debug)'
+        required: false
+      pre-built-kamel-image:
+        description: 'Kamel image url for skipping building of kamel stages. Used for debugging'
+        required: false
+      skip-problematic:
+        description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)'
+        required: false
+        default: false
+      test-filters:
+        description: |
+          Filter the tests in this test suite by assigning the test pattern to TEST_NATIVE_RUN,
+          eg. TEST_NATIVE_RUN=TestBasic will only run tests prefixed with 'TestBasic'
+        required: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  install-native-high-memory:
+
+    runs-on: macos-12
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+        submodules: recursive
+    - name: Convert input parameters to env vars
+      shell: bash
+      run: |
+        ./.github/workflows/manual-exec-process-inputs.sh \
+          -i "${{ github.event.inputs.pre-built-kamel-image }}" \
+          -p "${{ github.event.inputs.skip-problematic }}" \
+          -q "${{ github.event.inputs.log-level }}" \
+          -t "${{ github.event.inputs.test-filters }}"
+    - name: Native smoke tests
+      uses: ./.github/actions/e2e-install-native
+      with:
+        cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }}
+        cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }}
+        high-memory: 'true'
+
+  install-native:
+
+    runs-on: macos-12
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+        submodules: recursive
+    - name: Convert input parameters to env vars
+      shell: bash
+      run: |
+        ./.github/workflows/manual-exec-process-inputs.sh \
+          -i "${{ github.event.inputs.pre-built-kamel-image }}" \
+          -p "${{ github.event.inputs.skip-problematic }}" \
+          -q "${{ github.event.inputs.log-level }}" \
+          -t "${{ github.event.inputs.test-filters }}"
+    - name: Native smoke tests
+      uses: ./.github/actions/e2e-install-native
+      with:
+        cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }}
+        cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }}

config/crd/bases/camel.apache.org_builds.yaml

@@ -475,6 +475,67 @@ spec:
                           - provider
                           - version
                           type: object
+                        sources:
+                          description: the sources to add at build time
+                          items:
+                            description: SourceSpec defines the configuration for
+                              one or more routes to be executed in a certain Camel
+                              DSL language
+                            properties:
+                              compression:
+                                description: if the content is compressed (base64
+                                  encrypted)
+                                type: boolean
+                              content:
+                                description: the source code (plain text)
+                                type: string
+                              contentKey:
+                                description: the confimap key holding the source content
+                                type: string
+                              contentRef:
+                                description: the confimap reference holding the source
+                                  content
+                                type: string
+                              contentType:
+                                description: the content type (tipically text or binary)
+                                type: string
+                              interceptors:
+                                description: Interceptors are optional identifiers
+                                  the org.apache.camel.k.RoutesLoader uses to pre/post
+                                  process sources
+                                items:
+                                  type: string
+                                type: array
+                              language:
+                                description: specify which is the language (Camel
+                                  DSL) used to interpret this source code
+                                type: string
+                              loader:
+                                description: Loader is an optional id of the org.apache.camel.k.RoutesLoader
+                                  that will interpret this source at runtime
+                                type: string
+                              name:
+                                description: the name of the specification
+                                type: string
+                              path:
+                                description: the path where the file is stored
+                                type: string
+                              property-names:
+                                description: List of property names defined in the
+                                  source (e.g. if type is "template")
+                                items:
+                                  type: string
+                                type: array
+                              rawContent:
+                                description: the source code (binary)
+                                format: byte
+                                type: string
+                              type:
+                                description: Type defines the kind of source described
+                                  by this object
+                                type: string
+                            type: object
+                          type: array
                         steps:
                           description: the list of steps to execute (see pkg/builder/)
                           items:

config/crd/bases/camel.apache.org_camelcatalogs.yaml

@@ -305,6 +305,11 @@ spec:
                       items:
                         type: string
                       type: array
+                    metadata:
+                      additionalProperties:
+                        type: string
+                      description: the metadata of the loader
+                      type: object
                     version:
                       description: Maven Version
                       type: string