From a479eca7de120a270008ed37882de2e2cd814dbc Mon Sep 17 00:00:00 2001 From: Binbin Zhang Date: Tue, 14 Sep 2021 16:40:20 +0800 Subject: [PATCH 1/9] docs: Fix outdated links fix outdated links which were checked out by workflow/docs-url-alive-check Fixes #2630 Signed-off-by: Binbin Zhang --- docs/how-to/how-to-use-kata-containers-with-acrn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/how-to-use-kata-containers-with-acrn.md b/docs/how-to/how-to-use-kata-containers-with-acrn.md index 39abf4918..3bb038b9f 100644 --- a/docs/how-to/how-to-use-kata-containers-with-acrn.md +++ b/docs/how-to/how-to-use-kata-containers-with-acrn.md @@ -22,7 +22,7 @@ This document requires the presence of the ACRN hypervisor and Kata Containers o - ACRN supported [Hardware](https://projectacrn.github.io/latest/hardware.html#supported-hardware). > **Note:** Please make sure to have a minimum of 4 logical processors (HT) or cores. -- ACRN [software](https://projectacrn.github.io/latest/tutorials/kbl-nuc-sdc.html#use-the-script-to-set-up-acrn-automatically) setup. +- ACRN [software](https://projectacrn.github.io/latest/tutorials/run_kata_containers.html) setup. - For networking, ACRN supports either MACVTAP or TAP. If MACVTAP is not enabled in the Service OS, please follow the below steps to update the kernel: ```sh From 6b6d81ccedd45546246c01c5ef58d488839d8072 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 15 Oct 2021 13:10:37 +0800 Subject: [PATCH 2/9] runtime: kernel version with '+' as suffix panic in parse The current kernel version parse lib can't process suffix '+', as the modified kernel version will add '+' as suffix, thus panic will occur. For example, if the current kernel version is "5.14.0-rc4+", test TestHostNetworkingRequested will panic: --- FAIL: TestHostNetworkingRequested (0.00s) panic: &{DistroName:ubuntu DistroVersion:18.04 KernelVersion:5.11.0-rc3+ Issue: Passed:[] Failed:[] Debug:true ActualEUID:0}: failed to check test constraints: error: Build meta data is empty Here, remove the suffix '+' in kernel version fix helper. Fixes: #2809 Signed-off-by: Jianyong Wu --- src/runtime/pkg/katatestutils/constraints.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/katatestutils/constraints.go b/src/runtime/pkg/katatestutils/constraints.go index 5ac50de1d..d2f400b4b 100644 --- a/src/runtime/pkg/katatestutils/constraints.go +++ b/src/runtime/pkg/katatestutils/constraints.go @@ -147,8 +147,15 @@ func getDistroDetails() (name, version string, err error) { // centos: 3.10.0-957.12.1.el7.x86_64 // fedora: 5.0.9-200.fc29.x86_64 // +// For some self compiled kernel, the kernel version will be with "+" as its suffix +// For example: +// 5.12.0-rc4+ +// These kernel version can't be parsed by the current lib and lead to panic +// therefore the '+' should be removed. +// func fixKernelVersion(version string) string { - return strings.Replace(version, "_", "-", -1) + version = strings.Replace(version, "_", "-", -1) + return strings.Replace(version, "+", "", -1) } // handleDistroName checks that the current distro is compatible with From e97cd23bd6d9842e24bcc8dfaa10257632bcc41c Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 15 Oct 2021 13:25:26 +0800 Subject: [PATCH 3/9] runtime: current vcpu number should be limited The physical current vcpu number should not be used directly as the largest vcpu number is limited to defaultMaxQemuVCPUs. Here, a new helper is introduced in pkg/katautils/config.go to get current vcpu number. Fixes: #2809 Signed-off-by: Jianyong Wu --- src/runtime/pkg/katautils/config.go | 17 +++++++++++++++-- src/runtime/pkg/katautils/config_test.go | 17 ++++++++--------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index edf882c5f..1cceea8f3 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -307,11 +307,24 @@ func (h hypervisor) GetEntropySource() string { return h.EntropySource } +// Current cpu number should not larger than defaultMaxVCPUs() +func getCurrentCpuNum() uint32 { + var cpu uint32 + h := hypervisor{} + + cpu = uint32(goruntime.NumCPU()) + if cpu > h.defaultMaxVCPUs() { + cpu = h.defaultMaxVCPUs() + } + + return cpu +} + func (h hypervisor) defaultVCPUs() uint32 { - numCPUs := goruntime.NumCPU() + numCPUs := getCurrentCpuNum() if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) { - return uint32(numCPUs) + return numCPUs } if h.NumVCPUs == 0 { // or unspecified return defaultVCPUCount diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index e6df8985b..33a1056bf 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -14,7 +14,6 @@ import ( "path" "path/filepath" "reflect" - goruntime "runtime" "strings" "syscall" "testing" @@ -156,7 +155,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), HypervisorMachineType: machineType, NumVCPUs: defaultVCPUCount, - DefaultMaxVCPUs: uint32(goruntime.NumCPU()), + DefaultMaxVCPUs: getCurrentCpuNum(), MemorySize: defaultMemSize, DisableBlockDeviceUse: disableBlockDevice, BlockDeviceDriver: defaultBlockDeviceDriver, @@ -919,13 +918,13 @@ func TestNewClhHypervisorConfig(t *testing.T) { func TestHypervisorDefaults(t *testing.T) { assert := assert.New(t) - numCPUs := goruntime.NumCPU() + numCPUs := getCurrentCpuNum() h := hypervisor{} assert.Equal(h.machineType(), defaultMachineType, "default hypervisor machine type wrong") assert.Equal(h.defaultVCPUs(), defaultVCPUCount, "default vCPU number is wrong") - assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") assert.Equal(h.defaultMemSz(), defaultMemSize, "default memory size is wrong") machineType := "foo" @@ -934,23 +933,23 @@ func TestHypervisorDefaults(t *testing.T) { // auto inferring h.NumVCPUs = -1 - assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") + assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong") h.NumVCPUs = 2 assert.Equal(h.defaultVCPUs(), uint32(2), "default vCPU number is wrong") h.NumVCPUs = int32(numCPUs) + 1 - assert.Equal(h.defaultVCPUs(), uint32(numCPUs), "default vCPU number is wrong") + assert.Equal(h.defaultVCPUs(), numCPUs, "default vCPU number is wrong") h.DefaultMaxVCPUs = 2 assert.Equal(h.defaultMaxVCPUs(), uint32(2), "default max vCPU number is wrong") - h.DefaultMaxVCPUs = uint32(numCPUs) + 1 - assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + h.DefaultMaxVCPUs = numCPUs + 1 + assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") maxvcpus := vc.MaxQemuVCPUs() h.DefaultMaxVCPUs = maxvcpus + 1 - assert.Equal(h.defaultMaxVCPUs(), uint32(numCPUs), "default max vCPU number is wrong") + assert.Equal(h.defaultMaxVCPUs(), numCPUs, "default max vCPU number is wrong") h.MemorySize = 1024 assert.Equal(h.defaultMemSz(), uint32(1024), "default memory size is wrong") From 7cb650abcf1166054f75d3d3e5ed64b034ad8cb7 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 15 Oct 2021 13:33:19 +0800 Subject: [PATCH 4/9] runtime: DefaultMaxVCPUs should not greater than defaultMaxQemuVCPUs DefaultMaxVCPUs may be larger than the defaultMaxQemuVCPUs that should be checked and avoided. Fixes: #2809 Signed-off-by: Jianyong Wu --- src/runtime/virtcontainers/hypervisor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index bc92eb303..5d3295a19 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -517,7 +517,7 @@ func (conf *HypervisorConfig) valid() error { conf.BlockDeviceDriver = config.VirtioBlockCCW } - if conf.DefaultMaxVCPUs == 0 { + if conf.DefaultMaxVCPUs == 0 || conf.DefaultMaxVCPUs > defaultMaxQemuVCPUs { conf.DefaultMaxVCPUs = defaultMaxQemuVCPUs } From 0366f6e8177bd6e5e88130b909c263a3afac3c1d Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 15 Oct 2021 13:40:49 +0800 Subject: [PATCH 5/9] template: disable template unit test on arm Template is broken on arm. here we disable the template unit test temporarily. Fixes: #2809 Signed-off-by: Jianyong Wu --- src/runtime/virtcontainers/factory/template/template_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/factory/template/template_test.go b/src/runtime/virtcontainers/factory/template/template_test.go index c511baf92..e5a9e014a 100644 --- a/src/runtime/virtcontainers/factory/template/template_test.go +++ b/src/runtime/virtcontainers/factory/template/template_test.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "os" + "runtime" "testing" "time" @@ -22,7 +23,8 @@ import ( const testDisabledAsNonRoot = "Test disabled as requires root privileges" func TestTemplateFactory(t *testing.T) { - if os.Geteuid() != 0 { + // template is broken on arm64, so, temporarily disable it on arm64 + if runtime.GOARCH == "arm64" || os.Geteuid() != 0 { t.Skip(testDisabledAsNonRoot) } From 06d3049349c77fc88f0f724a75c75e3de3d3dd90 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 22 Oct 2021 21:05:30 +0800 Subject: [PATCH 6/9] agent: fix race condition when test watcher create_tmpfs won't pass as the race condition in watcher umount. quote James's words here: 1. Rust runs all tests in parallel. 2. Mounts are a process-wide, not a per-thread resource. The only test that calls watcher.mount() is create_tmpfs(). However, other tests create BindWatcher objects. 3. BindWatcher's drop() implementation calls self.cleanup(), which calls unmount for the mountpoint create_tmpfs() asserts. 4. The other tests are calling unmount whenever a BindWatcher goes out of scope. To avoid that issue, let the tests using BindWatcher in watcher and sandbox.rs run sequentially. Fixes: #2809 Signed-off-by: Jianyong Wu --- src/agent/Cargo.lock | 1 + src/agent/Cargo.toml | 1 + src/agent/src/sandbox.rs | 12 ++++++++++++ src/agent/src/watcher.rs | 5 +++++ 4 files changed, 19 insertions(+) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index f922acba9..3d873bf67 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -545,6 +545,7 @@ dependencies = [ "scan_fmt", "scopeguard", "serde_json", + "serial_test", "slog", "slog-scope", "slog-stdlog", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 10cf2082c..d1a4d5749 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -20,6 +20,7 @@ scan_fmt = "0.2.3" scopeguard = "1.0.0" thiserror = "1.0.26" regex = "1" +serial_test = "0.5.1" # Async helpers async-trait = "0.1.42" diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 93a4fa0fe..03a216219 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -465,7 +465,10 @@ mod tests { baremount.mount() } + use serial_test::serial; + #[tokio::test] + #[serial] async fn set_sandbox_storage() { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); @@ -500,6 +503,7 @@ mod tests { } #[tokio::test] + #[serial] async fn remove_sandbox_storage() { skip_if_not_root!(); @@ -556,6 +560,7 @@ mod tests { } #[tokio::test] + #[serial] async fn unset_and_remove_sandbox_storage() { skip_if_not_root!(); @@ -607,6 +612,7 @@ mod tests { } #[tokio::test] + #[serial] async fn unset_sandbox_storage() { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); @@ -690,6 +696,7 @@ mod tests { } #[tokio::test] + #[serial] async fn get_container_entry_exist() { skip_if_not_root!(); let logger = slog::Logger::root(slog::Discard, o!()); @@ -703,6 +710,7 @@ mod tests { } #[tokio::test] + #[serial] async fn get_container_no_entry() { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); @@ -712,6 +720,7 @@ mod tests { } #[tokio::test] + #[serial] async fn add_and_get_container() { skip_if_not_root!(); let logger = slog::Logger::root(slog::Discard, o!()); @@ -723,6 +732,7 @@ mod tests { } #[tokio::test] + #[serial] async fn update_shared_pidns() { skip_if_not_root!(); let logger = slog::Logger::root(slog::Discard, o!()); @@ -741,6 +751,7 @@ mod tests { } #[tokio::test] + #[serial] async fn add_guest_hooks() { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); @@ -764,6 +775,7 @@ mod tests { } #[tokio::test] + #[serial] async fn test_sandbox_set_destroy() { let logger = slog::Logger::root(slog::Discard, o!()); let mut s = Sandbox::new(&logger).unwrap(); diff --git a/src/agent/src/watcher.rs b/src/agent/src/watcher.rs index ef5030763..d35d591a4 100644 --- a/src/agent/src/watcher.rs +++ b/src/agent/src/watcher.rs @@ -982,7 +982,10 @@ mod tests { ); } + use serial_test::serial; + #[tokio::test] + #[serial] async fn create_tmpfs() { skip_if_not_root!(); @@ -997,6 +1000,7 @@ mod tests { } #[tokio::test] + #[serial] async fn spawn_thread() { skip_if_not_root!(); @@ -1026,6 +1030,7 @@ mod tests { } #[tokio::test] + #[serial] async fn verify_container_cleanup_watching() { skip_if_not_root!(); From 45f65a73c8feb28751f7aa6143854991060d3e12 Mon Sep 17 00:00:00 2001 From: Haitao Li Date: Sun, 24 Oct 2021 13:26:14 +1100 Subject: [PATCH 7/9] agent: Handle uevent remove actions uevents with action=remove was ignored causing the agent to reuse stale data in the device map. This patch adds handling of such uevents. Fixes #2405 Signed-off-by: Haitao Li --- src/agent/src/linux_abi.rs | 1 + src/agent/src/uevent.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index 08600d9f7..53fabaf0b 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -95,6 +95,7 @@ pub const SYSTEM_DEV_PATH: &str = "/dev"; // Linux UEvent related consts. pub const U_EVENT_ACTION: &str = "ACTION"; pub const U_EVENT_ACTION_ADD: &str = "add"; +pub const U_EVENT_ACTION_REMOVE: &str = "remove"; pub const U_EVENT_DEV_PATH: &str = "DEVPATH"; pub const U_EVENT_SUB_SYSTEM: &str = "SUBSYSTEM"; pub const U_EVENT_SEQ_NUM: &str = "SEQNUM"; diff --git a/src/agent/src/uevent.rs b/src/agent/src/uevent.rs index c5ecefcc1..84e1625f6 100644 --- a/src/agent/src/uevent.rs +++ b/src/agent/src/uevent.rs @@ -97,10 +97,18 @@ impl Uevent { }) } + #[instrument] + async fn process_remove(&self, logger: &Logger, sandbox: &Arc>) { + let mut sb = sandbox.lock().await; + sb.uevent_map.remove(&self.devpath); + } + #[instrument] async fn process(&self, logger: &Logger, sandbox: &Arc>) { if self.action == U_EVENT_ACTION_ADD { return self.process_add(logger, sandbox).await; + } else if self.action == U_EVENT_ACTION_REMOVE { + return self.process_remove(logger, sandbox).await; } debug!(*logger, "ignoring event"; "uevent" => format!("{:?}", self)); } From 4f73e58d73db7db6f74c41d38b1361573532b602 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 25 Oct 2021 18:47:35 +0200 Subject: [PATCH 8/9] packaging/static-build: s390x fixes - Install OpenSSL for key generation in kernel build - Do not install libpmem - Do not exclude `*/share/*/*.img` files in QEMU tarball since among them are boot loader files critical for IPLing. Fixes: #2895 Signed-off-by: Jakob Naucke --- tools/packaging/static-build/kernel/Dockerfile | 4 +++- tools/packaging/static-build/qemu.blacklist | 1 - tools/packaging/static-build/qemu/Dockerfile | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/packaging/static-build/kernel/Dockerfile b/tools/packaging/static-build/kernel/Dockerfile index b8b813396..40f3228f0 100644 --- a/tools/packaging/static-build/kernel/Dockerfile +++ b/tools/packaging/static-build/kernel/Dockerfile @@ -15,4 +15,6 @@ RUN apt install -y \ flex \ git \ iptables \ - libelf-dev \ + libelf-dev + +RUN [ "$(uname -m)" = "s390x" ] && apt-get install -y libssl-dev || true diff --git a/tools/packaging/static-build/qemu.blacklist b/tools/packaging/static-build/qemu.blacklist index 9c621893e..31f6ab23c 100644 --- a/tools/packaging/static-build/qemu.blacklist +++ b/tools/packaging/static-build/qemu.blacklist @@ -16,7 +16,6 @@ qemu_black_list=( */share/*/efi-rtl8139.rom */share/*/efi-vmxnet3.rom */share/*/icons -*/share/*/*.img */share/*/keymaps */share/*/multiboot.bin */share/*/npcm7xx_bootrom.bin diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index fbbb0dad6..cb15ddeac 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -36,7 +36,6 @@ RUN apt-get --no-install-recommends install -y \ libltdl-dev \ libmount-dev \ libpixman-1-dev \ - libpmem-dev \ libselinux1-dev \ libtool \ make \ @@ -49,6 +48,8 @@ RUN apt-get --no-install-recommends install -y \ rsync \ zlib1g-dev +RUN [ "$(uname -m)" != "s390x" ] && apt-get install -y libpmem-dev || true + ARG QEMU_REPO RUN cd .. && git clone "${QEMU_REPO}" qemu From b7493fd5d50795ff8e8008b582bd12f730df2b2e Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 5 Nov 2021 03:18:09 +0000 Subject: [PATCH 9/9] release: Kata Containers 2.2.3 ad45107a2 release: Kata Containers 2.2.3 4f73e58d7 packaging/static-build: s390x fixes 45f65a73c agent: Handle uevent remove actions 06d304934 agent: fix race condition when test watcher 0366f6e81 template: disable template unit test on arm 7cb650abc runtime: DefaultMaxVCPUs should not greater than defaultMaxQemuVCPUs e97cd23bd runtime: current vcpu number should be limited 6b6d81cce runtime: kernel version with '+' as suffix panic in parse a479eca7d docs: Fix outdated links b794a3940 virtcontainers: clh: Re-generate the client code 39d95f486 versions: Upgrade to Cloud Hypervisor v19.0 Depends-on: github.com/kata-containers/tests#4155 Signed-off-by: Peng Tao --- VERSION | 2 +- tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | 2 +- tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index b1b25a5ff..585940699 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2.2 +2.2.3 diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index 880bbe4e2..8a7f3d38a 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -18,7 +18,7 @@ spec: katacontainers.io/kata-runtime: cleanup containers: - name: kube-kata-cleanup - image: quay.io/kata-containers/kata-deploy:2.2.2 + image: quay.io/kata-containers/kata-deploy:2.2.3 imagePullPolicy: Always command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] env: diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index fd6e271fe..7808e5610 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -16,7 +16,7 @@ spec: serviceAccountName: kata-label-node containers: - name: kube-kata - image: quay.io/kata-containers/kata-deploy:2.2.2 + image: quay.io/kata-containers/kata-deploy:2.2.3 imagePullPolicy: Always lifecycle: preStop: