diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de3225..92cbbba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,7 @@ jobs: - name: Check deployment script syntax run: | make deploy-script-check + python -m unittest discover -s builder/tests -v standalone-e2e: name: Standalone E2E diff --git a/.gitmodules b/.gitmodules index 97ae8b8..6a898d0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/sandboxd"] path = src/sandboxd - url = https://github.com/inclusionAI/sandboxd.git + url = https://github.com/mhsong1998-dot/sandboxd.git [submodule "src/distill-fs"] path = src/distill-fs url = https://github.com/inclusionAI/distill-fs.git diff --git a/AGENTS.md b/AGENTS.md index f656f42..c84dbda 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -406,6 +406,13 @@ the sandbox bridge. YuanRong receives `INSTANCE_IP` in Kubernetes or the default-route interface address in standalone mode; `AKERNEL_NODE_IP` is the explicit override for multi-homed environments. +Node deployments keep openYuanRong's in-node address aligned with sandboxd's +actual bridge state. Kubernetes renders `node.sandboxIPRange` into sandboxd's +`ip_range`; standalone reads the same setting from `sandboxd_config.toml` when +preparing networking. The sandboxd systemd startup waits with a bounded +timeout for `sandbox0`; YuanRong starts afterward and passes the bridge's +assigned IPv4 address through `values.local_ip` in the Python CLI config. + The standalone sandboxd filestore is a loop-mounted ext4 image under the bind-mounted `deploy/standalone/data/` directory. Explicit `storage_mb` quotas for runsc and Firecracker use this local-disk filestore. Without an diff --git a/Makefile b/Makefile index 100218d..6e26177 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ deploy-script-check: @set -euo pipefail; \ while IFS= read -r -d '' script; do \ bash -n "$$script"; \ - done < <(git ls-files -z -- 'deploy/**/*.sh'); \ + done < <(git ls-files -z -- 'deploy/**/*.sh' 'builder/**/*.sh'); \ while IFS= read -r -d '' template; do \ if ! bash -n <( \ sed \ diff --git a/builder/config/openyuanrong-core-0.10.1rc4.constraints.txt b/builder/config/openyuanrong-core-0.10.1rc4.constraints.txt new file mode 100644 index 0000000..c623827 --- /dev/null +++ b/builder/config/openyuanrong-core-0.10.1rc4.constraints.txt @@ -0,0 +1,30 @@ +# Runtime dependency lock for openyuanrong-core 0.10.1rc4 on Python 3.12. +# Regenerate and validate this file for both linux/amd64 and linux/arm64 when +# OPEN_YR_VERSION changes. +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.14.2 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.4.2 +cloudpickle==3.1.2 +frozenlist==1.8.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +jinja2==3.1.6 +markupsafe==3.0.3 +msgpack==1.2.1 +multidict==6.7.1 +propcache==0.5.2 +protobuf==7.36.0 +pyyaml==6.0.3 +requests==2.34.2 +tomli-w==1.2.0 +typing-extensions==4.16.0 +urllib3==2.7.0 +websockets==17.0.1 +yarl==1.24.5 diff --git a/builder/config/yr/config.toml.jinja b/builder/config/yr/config.toml.jinja new file mode 100644 index 0000000..0123631 --- /dev/null +++ b/builder/config/yr/config.toml.jinja @@ -0,0 +1,193 @@ +{# + AKernel runtime profile template. + + Deployment values are resolved from the CLI-provided env mapping. Strings + containing Python CLI runtime expressions remain escaped for its second + rendering pass. +#} +{% macro toml_string(value) -%} +"{%- for char in value -%} +{%- set encoded = char | tojson -%} +{{- char if encoded | length == 14 else encoded[1:-1] -}} +{%- endfor -%}" +{%- endmacro %} +{% set role = env["AKERNEL_ROLE"] %} +{% set cluster_roles = ["master", "frontend"] %} +{% set node_roles = ["node", "standalone"] %} +{% if role not in cluster_roles + node_roles %} +{{ unsupported_akernel_role }} +{% endif %} +{% set is_cluster = role in cluster_roles %} +{% set is_node = role in node_roles %} +{% set is_agent = role == "node" %} +{% set is_standalone = role == "standalone" %} +{% set has_master_mode = not is_agent %} +{% set enable_metrics = env.get("ENABLE_METRICS", "false") | lower %} +{% set enable_trace = env.get("ENABLE_TRACE", "false") | lower %} +{% set traefik_enable_tls = "true" if env.get("TRAEFIK_ENABLE_TLS", "false") | lower == "true" else "false" %} +{% set configured_host_ip = env.get("YR_NODE_IP", "") | trim %} +{% set configured_local_ip = env.get("YR_LOCAL_IP", "") | trim %} +{% set enable_faas_frontend = env.get("ENABLE_FAAS_FRONTEND", "true") | lower %} +{% set enable_iam_server = env.get("ENABLE_IAM_SERVER", "true") | lower %} +{% set runtime_hostname = "{{ hostname }}" %} +{% set deploy_path_value = env["DEPLOY_PATH"] | trim %} +{% set component_log_path = env["YR_LOG_PATH"] | trim %} + +# Generated at startup from builder/config/yr/config.toml.jinja. + +[values] +node_id = "{{ runtime_hostname }}" +deploy_path = {{ toml_string(deploy_path_value) }} +{% if configured_host_ip %} +host_ip = {{ toml_string(configured_host_ip) }} +{% endif %} +{% if configured_local_ip %} +local_ip = {{ toml_string(configured_local_ip) }} +{% endif %} +{% if is_cluster %} +cpu_num = 1 +memory_num = 3904 +shared_memory_num = 4096 +{% endif %} + +[values.fs.log] +path = {{ toml_string(component_log_path) }} + +[values.fs.tls] +base_path = "/home/yuanrong/.cert" + +{% if not is_standalone %} +[values.etcd] +enable_multi_master = true +{% endif %} + +{% if is_cluster or is_node %} +[[values.etcd.address]] +ip = {{ toml_string(configured_host_ip if is_standalone else env["ETCD_ADDRESS"]) }} +port = {{ env.get("ETCD_PORT", "2379") }} +peer_port = {{ env.get("ETCD_PEER_PORT", "2378") }} +{% endif %} + +[values.frontend] +ssl_enable = true +client_auth_type = "NoClientCert" +frontend_lease_bypass = true +enable_function_token_auth = true +enable_func_token_auth = true +{% if role == "frontend" %} +meta_service_address = {{ toml_string(env["META_SERVICE_ADDRESS"]) }} +{% endif %} +iam_server_address = "127.0.0.1:31113" + +[values.meta_service] +port = 31111 + +{% if has_master_mode %} +[mode.master] +etcd = {{ "true" if is_standalone else "false" }} +ds_master = {{ "true" if is_standalone else "false" }} +frontend = {{ enable_faas_frontend if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +function_master = {{ "true" if role in ["master", "standalone"] else "false" }} +function_scheduler = false +meta_service = {{ "true" if role in ["master", "standalone"] else "false" }} +iam_server = {{ enable_iam_server if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +{% endif %} + +[ds_worker.args] +{% if is_cluster %} +rpc_thread_num = 128 +node_timeout_s = 10 +node_dead_timeout_s = 30 +heartbeat_interval_ms = 3000 +{% endif %} +{% if is_node %} +node_timeout_s = 30 +client_dead_timeout_s = 60 +heartbeat_interval_ms = 1000 +node_dead_timeout_s = 120 +log_dir = {{ toml_string(component_log_path ~ "/data_system/worker") }} +{% endif %} + +[ds_master.args] +log_dir = {{ toml_string(component_log_path ~ "/data_system/master") }} + +[function_master.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +traefik_enable_tls = {{ traefik_enable_tls }} +traefik_http_entry_point = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +traefik_forward_timeout_ms = 3000 +{% if enable_metrics == "true" and (is_cluster or is_standalone) %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_cluster or is_standalone %} +enable_traefik_provider = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "http" else "false" }} +{% endif %} +{% if is_cluster %} +system_timeout = 300000 +schedule_relaxed = 20 +{% elif is_standalone %} +system_timeout = 60000 +{% endif %} + +[function_proxy.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +enable_inherit_env = false +npu_collection_mode = "off" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +enable_direct_routing = false +force_low_reliability_instance = true +traefik_enable_tls = {{ traefik_enable_tls }} +{% if enable_metrics == "true" %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_node %} +enable_traefik_registry = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "etcd" else "false" }} +traefik_http_entrypoint = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +{% endif %} +log_expiration_enable = true +log_expiration_time_threshold = {{ 10 if is_agent else 7200 }} +log_expiration_cleanup_interval = {{ 10 if is_agent else 600 }} +log_expiration_max_file_count = {{ 50 if is_agent else 256 }} +{% if is_cluster %} +system_timeout = 300000 +pseudo_data_plane = true +{% elif is_node %} +system_timeout = 60000 +fc_agent_mgr_retry_times = 30 +fc_agent_mgr_retry_cycle = 60000 +{% endif %} +runtime_logs_dir = {{ toml_string(component_log_path) }} +{% if is_node %} +metrics_collector_type = "external" +snapshot_storage_mode = "local_only" +checkpoint_dir = "/home/akernel/checkpoints" +{% endif %} + +# The CLI exposes the meta-service component port separately from values.meta_service. +[meta_service] +port = 31111 + +[iam_server.args] +token_expired_time_span = 604800 +ssl_enable = true +iam_ssl_enable = true +local_listen_port = 31113 +local_ip = "127.0.0.1" +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +{% endif %} + +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +[frontend.env] +ENABLE_TRACE = "true" +TRACE_CONFIG = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 19669be..459cdb7 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -254,6 +254,7 @@ RUN apt-get update && \ procps \ python3 \ python3-pip \ + python3-venv \ systemd \ systemd-sysv \ tzdata \ @@ -296,6 +297,11 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ ENV YR_INSTALLATION_DIR=/home/yuanrong +ENV PATH=/opt/openyuanrong/bin:${PATH} + +COPY ./builder/config/yr/config.toml.jinja /etc/yuanrong/config.toml.jinja +COPY ./builder/config/openyuanrong-core-0.10.1rc4.constraints.txt /tmp/openyuanrong-core.constraints.txt +COPY ./builder/scripts/patch_openyuanrong_core.py /tmp/patch_openyuanrong_core.py # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the @@ -325,21 +331,36 @@ RUN set -eux; \ test -z "${OPEN_YR_CORE_WHEEL_SHA256}"; \ fi; \ wheel="/tmp/${wheel_name}"; \ - target=/tmp/openyuanrong-core; \ curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${wheel_url}" -o "${wheel}"; \ echo "${wheel_sha} ${wheel}" | sha256sum -c -; \ - python3 -m pip install \ - --break-system-packages \ + python3 -m venv /opt/openyuanrong; \ + /opt/openyuanrong/bin/python -m pip install \ --no-cache-dir \ - --no-deps \ - --target "${target}" \ + --index-url "${PIP_INDEX_URL}" \ + --constraint /tmp/openyuanrong-core.constraints.txt \ "${wheel}"; \ - test -x "${target}/yr/functionsystem/bin/yr"; \ - mkdir -p "${YR_INSTALLATION_DIR}"; \ - cp -a "${target}/yr/." "${YR_INSTALLATION_DIR}/"; \ - rm -rf "${target}" "${wheel}"; \ - ln -sfn "${YR_INSTALLATION_DIR}/functionsystem/bin/yr" /usr/bin/yr + site_packages="$(/opt/openyuanrong/bin/python -c 'import site; print(site.getsitepackages()[0])')"; \ + /opt/openyuanrong/bin/python /tmp/patch_openyuanrong_core.py "${site_packages}"; \ + base_py="${site_packages}/yr/cli/component/base.py"; \ + launcher_py="${site_packages}/yr/cli/system_launcher.py"; \ + grep -Fq 'logger.info(f"Environment: {full_env}")' "${base_py}"; \ + sed -i \ + 's/logger.info(f"Environment: {full_env}")/logger.info(f"Environment keys: {sorted(full_env)}")/' \ + "${base_py}"; \ + grep -Fq 'logger.info(f"Environment keys: {sorted(full_env)}")' "${base_py}"; \ + ! grep -Fq 'logger.info(f"Environment: {full_env}")' "${base_py}"; \ + grep -Fq '"env_vars": comp.env_vars,' "${launcher_py}"; \ + sed -i 's/"env_vars": comp.env_vars,/"env_vars": {},/' "${launcher_py}"; \ + grep -Fq '"env_vars": {},' "${launcher_py}"; \ + ! grep -Fq '"env_vars": comp.env_vars,' "${launcher_py}"; \ + test -x /opt/openyuanrong/bin/yr; \ + /opt/openyuanrong/bin/python -m py_compile \ + "${site_packages}/yr/cli/component/ds_worker.py"; \ + rm -f \ + "${wheel}" \ + /tmp/openyuanrong-core.constraints.txt \ + /tmp/patch_openyuanrong_core.py COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img @@ -358,6 +379,7 @@ RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then \ COPY ./builder/scripts/akernel-entrypoint.sh /usr/local/bin/akernel-entrypoint COPY ./builder/scripts/ensure-component-cert.sh /usr/local/bin/ensure-component-cert COPY ./builder/scripts/sandboxd_network_prepare.sh /usr/local/bin/sandboxd-network-prepare +COPY ./builder/scripts/sandboxd_network_ready.sh /usr/local/bin/sandboxd-network-ready RUN chmod 0755 \ /usr/local/bin/runsc \ /usr/local/bin/sandboxd \ @@ -366,7 +388,8 @@ RUN chmod 0755 \ /usr/local/bin/distill_fs \ /usr/local/bin/akernel-entrypoint \ /usr/local/bin/ensure-component-cert \ - /usr/local/bin/sandboxd-network-prepare + /usr/local/bin/sandboxd-network-prepare \ + /usr/local/bin/sandboxd-network-ready RUN if [ "${AKERNEL_ENABLE_KATA}" = "true" ]; then chmod 0755 /usr/local/bin/containerd-shim-kata-v2; fi RUN if [ "${AKERNEL_ENABLE_RUNC}" = "true" ]; then \ chmod 0755 /usr/local/bin/runc /usr/local/bin/runc-shim; \ diff --git a/builder/scripts/akernel-entrypoint.sh b/builder/scripts/akernel-entrypoint.sh index 1df691c..c134180 100644 --- a/builder/scripts/akernel-entrypoint.sh +++ b/builder/scripts/akernel-entrypoint.sh @@ -8,12 +8,8 @@ set -euo pipefail role="${AKERNEL_ROLE:-}" if [ -z "${role}" ] && [ "$#" -gt 0 ]; then - case "$1" in - master|frontend|node|standalone) - role="$1" - shift - ;; - esac + role="$1" + shift fi if [ -z "${role}" ]; then @@ -25,6 +21,16 @@ if [ -z "${role}" ]; then fi fi +case "${role}" in + master|frontend|node|standalone) + ;; + *) + echo "unsupported AKERNEL_ROLE: ${role}; expected master, frontend, node, or standalone" >&2 + exit 1 + ;; +esac +export AKERNEL_ROLE="${role}" + case "${role}" in master|frontend) /usr/local/bin/ensure-component-cert @@ -38,8 +44,4 @@ case "${role}" in /usr/local/bin/ensure-component-cert exec /usr/sbin/init "$@" ;; - *) - echo "unsupported AKERNEL_ROLE: ${role}" >&2 - exit 1 - ;; esac diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index 862d1b6..b0dda3d 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -3,46 +3,94 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 -set -e +set -euo pipefail ulimit -n 32768 -BASE_DIR=$( - cd "$(dirname "$0")" - pwd -) -export DEPLOY_PATH="/home/yuanrong/master/" -mkdir -p "$DEPLOY_PATH" -export YR_LOG_PATH="$DEPLOY_PATH/log" + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" +export DEPLOY_PATH="${DEPLOY_PATH:-/home/yuanrong/master}" +# The all-in-one image supplies the node log path; master/frontend use their +# own fixed role directory, as in the legacy launcher. +export YR_LOG_PATH="${DEPLOY_PATH}/log" export YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" -# If ConfigMap-mounted config exists, symlink it to override the baked-in default -[ -f /etc/otel-collector/otel_config.yaml ] && ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml +if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 + exit 1 +fi -# otel watchdog: monitor and restart otelcol-contrib if it crashes +if [ ! -x "${YR_CLI}" ]; then + echo "yr binary not found or not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 + exit 1 + fi + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT +else + unset YR_TRACE_CONFIG_CONTENT +fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start --master --block true + --port-policy FIX + --function-proxy-merge-process-enable +) +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${DEPLOY_PATH}" "${YR_LOG_PATH}" + +# If ConfigMap-mounted config exists, symlink it to override the baked-in default. +[ -f /etc/otel-collector/otel_config.yaml ] && + ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml + +# Monitor and restart the collector when observability is enabled. otel_watchdog() { - local otel_log="$DEPLOY_PATH/otelcol.log" + local otel_log="${DEPLOY_PATH}/otelcol.log" local max_restart_interval=60 local restart_count=0 while true; do - otelcol-contrib --config="/home/yuanrong/otel_config.yaml" >> "$otel_log" 2>&1 & + otelcol-contrib --config=/home/yuanrong/otel_config.yaml >> "${otel_log}" 2>&1 & local otel_pid=$! - echo $otel_pid > $DEPLOY_PATH/otelcol.pid - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: $otel_pid (restart count: $restart_count)" >> "$otel_log" + echo "${otel_pid}" > "${DEPLOY_PATH}/otelcol.pid" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: ${otel_pid} (restart count: ${restart_count})" >> "${otel_log}" - wait $otel_pid - local exit_code=$? + local exit_code=0 + wait "${otel_pid}" || exit_code=$? restart_count=$((restart_count + 1)) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code $exit_code, restarting in 5s (restart count: $restart_count)" >> "$otel_log" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code ${exit_code}, restarting (restart count: ${restart_count})" >> "${otel_log}" - # Exponential backoff with a cap local delay=$((2 ** restart_count)) - [ $delay -gt $max_restart_interval ] && delay=$max_restart_interval - sleep $delay + [ "${delay}" -gt "${max_restart_interval}" ] && delay="${max_restart_interval}" + sleep "${delay}" done } export -f otel_watchdog -if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && command -v otelcol-contrib >/dev/null 2>&1; then +if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && + command -v otelcol-contrib >/dev/null 2>&1; then nohup bash -c otel_watchdog & echo "otelcol watchdog started" echo "otel log: ${DEPLOY_PATH}/otelcol.log" @@ -50,65 +98,4 @@ else echo "otelcol watchdog skipped" fi -# Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then - ENABLE_TRAEFIK_PROVIDER=true -else - ENABLE_TRAEFIK_PROVIDER=false -fi - -if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 - exit 1 -fi - -YR_BIN="${YR_BIN:-/usr/bin/yr}" -if [ ! -x "${YR_BIN}" ]; then - echo "yr binary not found or not executable: ${YR_BIN}" >&2 - exit 1 -fi - -exec "${YR_BIN}" start --master --block true \ - -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ - -d $DEPLOY_PATH \ - --fs_health_check_retry_interval 1 \ - --schedule_relaxed 20 \ - --enable_faas_frontend ${ENABLE_FAAS_FRONTEND:-true} \ - --enable_function_scheduler ${ENABLE_FUNCTION_SCHEDULER:-false} \ - --enable_meta_service ${ENABLE_META_SERVICE:-true} \ - --enable_iam_server ${ENABLE_IAM_SERVER:-true} \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth ${ENABLE_FUNCTION_TOKEN_AUTH:-true} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --port_policy FIX \ - --system_timeout 300000 \ - --enable_distributed_master false \ - --etcd_mode outter \ - --etcd_addr_list $ETCD_ADDRESS \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port 2378 \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --ds_rpc_thread_num 128 \ - --function_proxy_merge_process_enable true \ - --force_low_reliability_instance true \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_forward_timeout_ms=3000 \ - --frontend_lease_bypass true \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --enable_direct_routing false \ - --enable_sandbox_router true \ - ${META_SERVICE_ADDRESS:+--meta_service_address $META_SERVICE_ADDRESS} +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/scripts/patch_openyuanrong_core.py b/builder/scripts/patch_openyuanrong_core.py new file mode 100644 index 0000000..57b62c7 --- /dev/null +++ b/builder/scripts/patch_openyuanrong_core.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Apply AKernel compatibility fixes to the pinned openYuanRong core wheel.""" + +from pathlib import Path +import sys + + +RELATIVE_SOURCE = Path("yr/cli/component/ds_worker.py") +OLD = " dirs = [rocksdb_dir, socket_dir, logs_dir]\n" +NEW = """ health_check_dir = Path( + self.resolver.rendered_config[self.name]["health_check"]["endpoint"] + ).parent + dirs = [rocksdb_dir, socket_dir, logs_dir, health_check_dir] +""" + + +def patch(site_packages: Path) -> None: + source = site_packages / RELATIVE_SOURCE + contents = source.read_text() + if NEW in contents: + if OLD in contents: + raise RuntimeError(f"ambiguous partially patched source: {source}") + return + if contents.count(OLD) != 1: + raise RuntimeError(f"expected ds_worker directory setup not found once: {source}") + source.write_text(contents.replace(OLD, NEW, 1)) + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} SITE_PACKAGES", file=sys.stderr) + return 2 + patch(Path(sys.argv[1])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/builder/scripts/sandboxd_network_ready.sh b/builder/scripts/sandboxd_network_ready.sh new file mode 100755 index 0000000..81a1338 --- /dev/null +++ b/builder/scripts/sandboxd_network_ready.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +for ((attempt = 0; attempt < 60; attempt++)); do + address="$( + { ip -4 -o address show dev sandbox0 scope global 2>/dev/null || true; } | + awk 'NR == 1 { split($4, value, "/"); print value[1] }' + )" + if [ -n "${address}" ]; then + echo "sandbox0 is ready with IPv4 address ${address}" + exit 0 + fi + sleep 1 +done + +echo "timed out after 60s waiting for sandbox0 to have an IPv4 address" >&2 +exit 1 diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 4b8b8e4..bfbb65c 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -3,7 +3,13 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + ulimit -n 32768 + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" export YR_RUNTIME_BACKEND=sandboxd export YR_IMAGE_PROCESS_CONFIG="${YR_IMAGE_PROCESS_CONFIG:-/run/akernel/yr-image-process.json}" @@ -40,117 +46,111 @@ resolve_node_ip() { printf '%s\n' "${node_ip}" } +resolve_sandbox_ip() { + local address + + address="$( + ip -4 -o address show dev sandbox0 scope global 2>/dev/null | + awk 'NR == 1 { split($4, value, "/"); print value[1] }' + )" + if [ -z "${address}" ]; then + echo "sandbox0 has no global IPv4 address after sandboxd startup" >&2 + return 1 + fi + printf '%s\n' "${address}" +} + YR_NODE_IP="$(resolve_node_ip)" +export YR_NODE_IP echo "Using ${YR_NODE_IP} as the YuanRong node address" CHECKPOINT_DIR="/home/akernel/checkpoints" mkdir -p "${CHECKPOINT_DIR}" -# Select the legacy etcd registry or the FunctionMaster HTTP provider. -if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then - ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} - ENABLE_TRAEFIK_PROVIDER=false -else - ENABLE_TRAEFIK_REGISTRY=false - ENABLE_TRAEFIK_PROVIDER=true -fi +YR_LOCAL_IP="$(resolve_sandbox_ip)" +export YR_LOCAL_IP +echo "Using ${YR_LOCAL_IP} as the YuanRong sandbox-local service address" -if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then - if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then - LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" - export LITEBUS_DATA_KEY +role="${AKERNEL_ROLE:-}" +if [ -z "${role}" ]; then + if [ "${AKS_LOCAL_MODE:-false}" = "true" ]; then + role=standalone + else + role=node fi - if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + export AKERNEL_ROLE="${role}" +fi + +case "${role}" in + node) + ;; + standalone) + if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then + LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" + export LITEBUS_DATA_KEY + fi + if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + exit 1 + fi + ;; + *) + echo "AKERNEL_ROLE must be node or standalone" >&2 + exit 1 + ;; +esac + +export DEPLOY_PATH="${DEPLOY_PATH:-/home/yuanrong}" +export YR_LOG_PATH="${YR_LOG_PATH:-${DEPLOY_PATH}/logs}" + +if [ ! -f "${YR_CONFIG_TEMPLATE}" ]; then + echo "YuanRong CLI config template not found: ${YR_CONFIG_TEMPLATE}" >&2 + exit 1 +fi +if [ ! -x "${YR_CLI}" ]; then + echo "YuanRong CLI not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 exit 1 fi - /usr/bin/yr start --master \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --enable_function_scheduler=false \ - --enable_faas_frontend=true \ - --enable_meta_service=true \ - --enable_iam_server=true \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth true \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --system_timeout 60000 \ - --block true \ - --etcd_port ${ETCD_PORT:-2379} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --log_root "${YR_LOG_PATH}" \ - --function_proxy_merge_process_enable true \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --frontend_lease_bypass true \ - --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ - --enable_sandbox_router true \ - --enable_direct_routing false + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT else - /usr/bin/yr start \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --etcd_addr_list ${ETCD_ADDRESS} \ - --etcd_mode outter \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --system_timeout 60000 \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - -n ${HOSTNAME} \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --log_root "${YR_LOG_PATH}" \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --log_expiration_time_threshold 10 \ - --log_expiration_cleanup_interval 10 \ - --log_expiration_max_file_count 50 \ - --function_proxy_merge_process_enable true \ - --enable_direct_routing false \ - --force_low_reliability_instance true \ - --snapshot_storage_mode local_only \ - --checkpoint_dir "${CHECKPOINT_DIR}" \ - --block true + unset YR_TRACE_CONFIG_CONTENT fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start +) +if [ "${role}" = "standalone" ]; then + YR_CLI_ARGS+=(--master) +fi +YR_CLI_ARGS+=( + --block true + --port-policy FIX + --function-proxy-merge-process-enable +) + +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${YR_LOG_PATH:-/home/yuanrong/logs}" +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/systemd_services/sandboxd.service b/builder/systemd_services/sandboxd.service index 7642142..5a366a1 100644 --- a/builder/systemd_services/sandboxd.service +++ b/builder/systemd_services/sandboxd.service @@ -4,7 +4,7 @@ After=network-online.target Before=yuanrong.service [Service] -PIDFile=/var/run/sandboxd.pid +Type=simple PassEnvironment=NODE_NAME KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT Environment=GOGC=300 @@ -17,6 +17,7 @@ ExecStart=/bin/bash -ce "exec /usr/local/bin/sandboxd \ -log-level debug \ -log-file /home/akernel/logs/sandboxd/sandboxd.log \ -config /home/akernel/sandboxd/config.toml" +ExecStartPost=/usr/local/bin/sandboxd-network-ready KillMode=process Restart=always diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 4572f5b..34d9bbd 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -1,10 +1,12 @@ [Unit] Description=yuanrong.service +Requires=sandboxd.service +After=sandboxd.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=AKERNEL_ROLE AKERNEL_NODE_IP INSTANCE_IP ETCD_ADDRESS ETCD_PORT ETCD_PEER_PORT HOSTNAME AKS_LOCAL_MODE LITEBUS_DATA_KEY DEPLOY_PATH YR_LOG_PATH YR_INSTALLATION_DIR YR_RRT_CONTROL_SOCKET_PATH YR_IMAGE_PROCESS_CONFIG ENABLE_METRICS ENABLE_TRACE TRACE_CONFIG_FILE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" @@ -14,6 +16,7 @@ ExecStart=/usr/bin/bash /home/yuanrong/yr_node_bootstrap.sh ExecReload=/bin/kill -15 $MAINPID KillMode=control-group Restart=always +RestartSec=5s Delegate=yes UMask=000 TasksMax=infinity diff --git a/builder/tests/test_master_entrypoint.py b/builder/tests/test_master_entrypoint.py new file mode 100644 index 0000000..826c53f --- /dev/null +++ b/builder/tests/test_master_entrypoint.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "master_entrypoint.sh" + + +class MasterEntrypointTest(unittest.TestCase): + def test_role_log_path_overrides_node_image_default(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cli = root / "yr" + cli.write_text('#!/bin/sh\nprintf "%s" "$YR_LOG_PATH" > "$LOG_CAPTURE"\n') + cli.chmod(0o755) + script = root / "entrypoint.sh" + # Replace only the installed CLI location; run the actual bootstrap. + script.write_text(SCRIPT.read_text().replace( + "YR_CLI=/opt/openyuanrong/bin/yr", 'YR_CLI="{}"'.format(cli), 1 + )) + for role in ("master", "frontend"): + with self.subTest(role=role): + environment = os.environ.copy() + environment.pop("DEPLOY_PATH", None) + environment.update( + AKERNEL_ROLE=role, + LITEBUS_DATA_KEY="test-seed", + YR_LOG_PATH="/home/yuanrong/logs", + ENABLE_TRACE="false", + YR_RENDERED_CONFIG_PATH=str(root / "config.toml"), + YR_CLI_DRY_RUN="true", + YR_CLI_CAPTURE_FILE=str(root / "args"), + LOG_CAPTURE=str(root / "log-path"), + ) + result = subprocess.run( + ["bash", str(script)], env=environment, + capture_output=True, text=True, timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + (root / "log-path").read_text(), + "/home/yuanrong/master/log", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/builder/tests/test_patch_openyuanrong_core.py b/builder/tests/test_patch_openyuanrong_core.py new file mode 100644 index 0000000..61bddae --- /dev/null +++ b/builder/tests/test_patch_openyuanrong_core.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "patch_openyuanrong_core.py" +RELATIVE_SOURCE = Path("yr/cli/component/ds_worker.py") + + +class PatchOpenYuanRongCoreTest(unittest.TestCase): + def test_creates_ds_worker_health_check_parent(self): + with tempfile.TemporaryDirectory() as directory: + site_packages = Path(directory) + source = site_packages / RELATIVE_SOURCE + source.parent.mkdir(parents=True) + source.write_text( + " dirs = [rocksdb_dir, socket_dir, logs_dir]\n" + ) + + for _ in range(2): + result = subprocess.run( + ["python3", str(SCRIPT), str(site_packages)], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + contents = source.read_text() + self.assertIn('["health_check"]["endpoint"]', contents) + self.assertIn( + "dirs = [rocksdb_dir, socket_dir, logs_dir, health_check_dir]", + contents, + ) + + def test_rejects_an_unknown_wheel_layout(self): + with tempfile.TemporaryDirectory() as directory: + site_packages = Path(directory) + source = site_packages / RELATIVE_SOURCE + source.parent.mkdir(parents=True) + source.write_text("unexpected source\n") + + result = subprocess.run( + ["python3", str(SCRIPT), str(site_packages)], + capture_output=True, + text=True, + timeout=10, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected ds_worker directory setup", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/builder/tests/test_sandboxd_network_ready.py b/builder/tests/test_sandboxd_network_ready.py new file mode 100644 index 0000000..5c064cf --- /dev/null +++ b/builder/tests/test_sandboxd_network_ready.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import textwrap +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "sandboxd_network_ready.sh" + + +class SandboxdNetworkReadyTest(unittest.TestCase): + def run_script(self, ip_mode): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + bin_path = temp_path / "bin" + bin_path.mkdir() + state_path = temp_path / "ip-count" + sleep_path = temp_path / "sleep-count" + self.write_mock( + bin_path / "ip", + """\ + #!/bin/sh + count=0 + if [ -f "$MOCK_IP_STATE" ]; then + count=$(cat "$MOCK_IP_STATE") + fi + count=$((count + 1)) + echo "$count" > "$MOCK_IP_STATE" + case "$MOCK_IP_MODE" in + missing-then-ready) + if [ "$count" -eq 1 ]; then + exit 1 + fi + ;; + no-ipv4) + exit 0 + ;; + esac + echo "2: sandbox0 inet 192.0.2.10/24 scope global sandbox0" + """, + ) + self.write_mock( + bin_path / "sleep", + """\ + #!/bin/sh + count=0 + if [ -f "$MOCK_SLEEP_STATE" ]; then + count=$(cat "$MOCK_SLEEP_STATE") + fi + echo $((count + 1)) > "$MOCK_SLEEP_STATE" + """, + ) + environment = os.environ.copy() + environment.update( + { + "PATH": "{}:{}".format(bin_path, environment["PATH"]), + "MOCK_IP_MODE": ip_mode, + "MOCK_IP_STATE": str(state_path), + "MOCK_SLEEP_STATE": str(sleep_path), + } + ) + result = subprocess.run( + ["bash", str(SCRIPT)], + capture_output=True, + env=environment, + text=True, + ) + ip_calls = int(state_path.read_text()) + sleep_calls = int(sleep_path.read_text()) if sleep_path.exists() else 0 + return result, ip_calls, sleep_calls + + @staticmethod + def write_mock(path, contents): + path.write_text(textwrap.dedent(contents)) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + def test_missing_interface_then_ready(self): + result, ip_calls, sleep_calls = self.run_script("missing-then-ready") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("sandbox0 is ready with IPv4 address 192.0.2.10", result.stdout) + self.assertEqual(ip_calls, 2) + self.assertEqual(sleep_calls, 1) + + def test_no_ipv4_until_timeout(self): + result, ip_calls, sleep_calls = self.run_script("no-ipv4") + + self.assertEqual(result.returncode, 1) + self.assertIn("timed out after 60s", result.stderr) + self.assertEqual(ip_calls, 60) + self.assertEqual(sleep_calls, 60) + + def test_immediately_ready(self): + result, ip_calls, sleep_calls = self.run_script("ready") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("sandbox0 is ready with IPv4 address 192.0.2.10", result.stdout) + self.assertEqual(ip_calls, 1) + self.assertEqual(sleep_calls, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..472555a 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -94,18 +94,8 @@ spec: secretKeyRef: name: {{ include "core.litebusSecretName" . }} key: litebus-data-key - - name: ENABLE_FUNCTION_MASTER - value: "false" - - name: ENABLE_FUNCTION_SCHEDULER - value: "false" - - name: ENABLE_META_SERVICE - value: "false" - - name: ENABLE_IAM_SERVER - value: "true" - name: META_SERVICE_ADDRESS value: {{ .Values.frontend.master.metaServiceAddress | default (printf "akernel-master.%s.svc.cluster.local:31111" .Release.Namespace) | quote }} - - name: IAM_SERVER_ADDRESS - value: {{ .Values.frontend.master.iamServerAddress | default (printf "akernel-master.%s.svc.cluster.local:31112" .Release.Namespace) | quote }} - name: ETCD_ADDRESS value: {{ get $frontendEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT @@ -124,13 +114,25 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} ports: - name: http containerPort: 8888 + startupProbe: + httpGet: + path: /healthz + port: 8888 + scheme: HTTPS + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 livenessProbe: httpGet: path: /healthz diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index de37591..9f873fe 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -108,16 +108,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} ports: - name: http containerPort: 8888 @@ -125,6 +133,12 @@ spec: containerPort: 31111 - name: global-sched containerPort: 22770 + startupProbe: + tcpSocket: + port: 22770 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 livenessProbe: tcpSocket: port: 22770 diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index bfc8989..b2d643f 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -13,7 +13,7 @@ data: CPUQuota={{ .Values.node.config.resourceControl.cpuQuota | default "800%" }} sandboxd_config.toml: | -{{- $sandboxdConfig := .Values.node.config.sandboxd.config }} +{{- $sandboxdConfig := tpl .Values.node.config.sandboxd.config . }} {{- $runcMarker := "# AKERNEL_RUNTIME_RUNC" }} {{- if .Values.node.config.sandboxd.enableRunc }} {{- if not (contains $runcMarker $sandboxdConfig) }} diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 01eb11d..f3b7041 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -59,6 +59,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- $nodeEtcd := .Values.node.etcd | default dict }} + {{- $sandboxResolverIP := first (splitList "/" .Values.node.sandboxIPRange) }} containers: - name: akernel-node image: "{{ include "core.image" (dict "root" . "image" .Values.node.image) }}" @@ -72,7 +73,11 @@ spec: - bash - -c - | - cp /etc/resolv.conf /etc/resolv_akernel.conf && sed -i 's/127.0.0.1/10.88.0.1/g' /etc/resolv_akernel.conf + sandbox_resolver_ip={{ $sandboxResolverIP | quote }} + awk -v resolver="${sandbox_resolver_ip}" ' + $1 == "nameserver" && $2 ~ /^127\./ { $2 = resolver } + { print } + ' /etc/resolv.conf > /etc/resolv_akernel.conf resources: {{- toYaml .Values.node.resources | nindent 10 }} env: @@ -122,16 +127,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} securityContext: privileged: true volumeMounts: diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index ab638fd..a94da49 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -123,6 +123,11 @@ frontend: iamServerAddress: "" node: + # sandboxd allocates sandbox addresses from this CIDR. + sandboxIPRange: "10.88.0.1/16" + etcd: + host: "" + port: "2379" secret: create: true oss_auths: {} @@ -406,13 +411,14 @@ node: } oss: - type: oss - oss: + type: s3 + s3: scheme: "http" endpoint: "" + region: "us-east-1" access_key_id: "" access_key_secret: "" - bucket: "" + bucket_name: "" object_prefix: "/" retry_limit: 3 connect_timeout: 300 @@ -459,7 +465,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="iptables" enable_network_acl=true diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0ce6d32..51359ac 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -77,6 +77,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "10.88.0.1/16" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -156,7 +157,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" enable_network_acl=true diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index da5e56b..5eeff98 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -39,8 +39,10 @@ locals { } : {} oss_auths = merge(local.generated_oss_auths, var.oss_auths) registry_auths = { - auths = { for host, cred in var.registry_auths : host => { username = cred.username, password = cred.password } } + auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } } } + registry_auths_enabled = length(var.registry_auths) > 0 + dockerconfigjson = base64encode(jsonencode(local.registry_auths)) # When auto-creating ELB on Huawei Cloud CCE, inject required annotations # so the cloud-controller-manager provisions the ELB automatically. @@ -119,6 +121,8 @@ locals { node_home_csi_size = var.node_home_csi_size oss_auths = local.oss_auths registry_auths = local.registry_auths + registry_auths_enabled = local.registry_auths_enabled + dockerconfigjson = local.dockerconfigjson etcd_cpu = var.etcd_resources.cpu etcd_memory = var.etcd_resources.memory diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 89c3602..7fa2941 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -1,3 +1,13 @@ +%{ if registry_auths_enabled ~} +imagePullSecrets: + - name: registry-secret + +imagePullSecret: + create: true + name: "registry-secret" + dockerconfigjson: "${dockerconfigjson}" +%{ endif ~} + kruise: enabled: ${enable_kruise} @@ -68,6 +78,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "10.88.0.1/16" image: repository: "${node_image_repository}" tag: "${node_image_tag}" @@ -136,7 +147,7 @@ node: stream_server_port="" [plugin.network] - ip_range="10.88.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" enable_network_acl=true diff --git a/src/sandboxd b/src/sandboxd index b892414..bf42f63 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit b892414a3b21f3c2a4481322767362af14fc701f +Subproject commit bf42f63b2916afdcb6414374469df713c8209381