systemvm-template: support on-demand download during setup and registration

client/bindir/cloud-setup-management.in

@@ -36,6 +36,100 @@ from cloudutils.cloudException import CloudRuntimeException, CloudInternalExcept
 from cloudutils.globalEnv import globalEnv
 from cloudutils.serviceConfigServer import cloudManagementConfig
 from optparse import OptionParser
+import urllib.request
+import configparser
+import hashlib
+
+SYSTEMVM_TEMPLATES_PATH = "/usr/share/cloudstack-management/templates/systemvm"
+SYSTEMVM_TEMPLATES_METADATA_FILE = SYSTEMVM_TEMPLATES_PATH + "/metadata.ini"
+
+def verify_sha512_checksum(file_path, expected_checksum):
+    sha512 = hashlib.sha512()
+    try:
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                sha512.update(chunk)
+        return sha512.hexdigest().lower() == expected_checksum.lower()
+    except Exception as e:
+        print(f"Failed to verify checksum for {file_path}: {e}")
+        return False
+
+def download_file(url, dest_path, chunk_size=8 * 1024 * 1024):
+    """
+    Downloads a file from the given URL to the specified destination path in chunks.
+    """
+    try:
+        with urllib.request.urlopen(url) as response:
+            total_size = response.length if response.length else None
+            downloaded = 0
+            try:
+                with open(dest_path, 'wb') as out_file:
+                    while True:
+                        chunk = response.read(chunk_size)
+                        if not chunk:
+                            break
+                        out_file.write(chunk)
+                        downloaded += len(chunk)
+                        if total_size:
+                            print(f"Downloaded {downloaded / (1024 * 1024):.2f}MB of {total_size / (1024 * 1024):.2f}MB", end='\r')
+            except PermissionError as pe:
+                print(f"Permission denied: {dest_path}")
+                raise
+        print(f"\nDownloaded file from {url} to {dest_path}")
+    except Exception as e:
+        print(f"Failed to download file: {e}")
+        raise
+
+def download_template_if_needed(template, url, filename, checksum):
+    dest_path = os.path.join(SYSTEMVM_TEMPLATES_PATH, filename)
+    if os.path.exists(dest_path):
+        if checksum and verify_sha512_checksum(dest_path, checksum):
+            print(f"{template} System VM template already exists at {dest_path} with valid checksum, skipping download.")
+            return
+        else:
+            print(f"{template} System VM template at {dest_path} has invalid or missing checksum, re-downloading...")
+    else:
+        print(f"Downloading {template} System VM template from {url} to {dest_path}...")
+    try:
+        download_file(url, dest_path)
+    except Exception as e:
+        print(f"ERROR: Failed to download {template} System VM template: {e}")
+
+def collect_template_metadata(selected_templates, options):
+    template_metadata_list = []
+    if not os.path.exists(SYSTEMVM_TEMPLATES_METADATA_FILE):
+        print(f"ERROR: System VM templates metadata file not found at {SYSTEMVM_TEMPLATES_METADATA_FILE}, cannot download templates.")
+        sys.exit(1)
+    config = configparser.ConfigParser()
+    config.read(SYSTEMVM_TEMPLATES_METADATA_FILE)
+    template_repo_url = None
+    if options.systemvm_templates_repository:
+        if "default" in config and "downloadrepository" in config["default"]:
+            template_repo_url = config["default"]["downloadrepository"].strip()
+        if not template_repo_url:
+            print("ERROR: downloadrepository value is empty in metadata file, cannot use --systemvm-template-repository option.")
+            sys.exit(1)
+    for template in selected_templates:
+        if template in config:
+            url = config[template].get("downloadurl")
+            filename = config[template].get("filename")
+            checksum = config[template].get("checksum")
+            if url and filename:
+                if template_repo_url:
+                    url = url.replace(template_repo_url, options.systemvm_templates_repository)
+                template_metadata_list.append({
+                    "template": template,
+                    "url": url,
+                    "filename": filename,
+                    "checksum": checksum
+                })
+            else:
+                print(f"ERROR: URL or filename not found for {template} System VM template in metadata.")
+                sys.exit(1)
+        else:
+            print(f"ERROR: No metadata found for {template} System VM template.")
+            sys.exit(1)
+    return template_metadata_list
 
 if __name__ == '__main__':
     initLoging("@MSLOGDIR@/setupManagement.log")
@@ -45,6 +139,16 @@ if __name__ == '__main__':
     parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server")
     parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it")
     parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration")
+    parser.add_option(
+        "--systemvm-templates",
+        dest="systemvm_templates",
+        help="Specify System VM templates to download: all, kvm-aarch64, kvm-x86_64, xenserver, vmware or comma-separated list of hypervisor combinations (e.g., kvm-x86_64,xenserver). Default is kvm-x86_64.",
+    )
+    parser.add_option(
+        "--systemvm-templates-repository",
+        dest="systemvm_templates_repository",
+        help="Specify the URL to download System VM templates from."
+    )
     (options, args) = parser.parse_args()
     if options.https:
         glbEnv.svrMode = "HttpsServer"
@@ -53,6 +157,22 @@ if __name__ == '__main__':
     if options.nostart:
         glbEnv.noStart = True
 
+    available_templates = ["kvm-aarch64", "kvm-x86_64", "xenserver", "vmware"]
+    templates_arg = options.systemvm_templates
+
+    selected_templates = ["kvm-x86_64"]
+    if templates_arg:
+        templates_list = [t.strip().lower() for t in templates_arg.split(",")]
+        if "all" in templates_list:
+            selected_templates = available_templates
+        else:
+            selected_templates = [t for t in templates_list if t in available_templates]
+    print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}")
+
+    template_metadata_list = []
+    if selected_templates:
+        template_metadata_list = collect_template_metadata(selected_templates, options)
+
     glbEnv.mode = "Server"
 
     print("Starting to configure CloudStack Management Server:")
@@ -68,9 +188,13 @@ if __name__ == '__main__':
         print("CloudStack Management Server setup is Done!")
         print("Please ensure ports 8080, 8250, 8443, and 9090 are opened and not firewalled for the management server and not in use by other processes on this host.")
     except (CloudRuntimeException, CloudInternalException) as e:
+
         print(e)
         print("Try to restore your system:")
         try:
             syscfg.restore()
         except:
             pass
+
+    for meta in template_metadata_list:
+        download_template_if_needed(meta["template"], meta["url"], meta["filename"], meta["checksum"])

client/conf/server.properties.in

@@ -62,3 +62,8 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@
 # Thread pool configuration
 #threads.min=10
 #threads.max=500
+
+# The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the
+# `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL
+# will be for download.
+# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/4.20/

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -108,7 +108,10 @@ public class SystemVmTemplateRegistration {
     private static Integer LINUX_12_ID = 363;
     private static final Integer SCRIPT_TIMEOUT = 1800000;
     private static final Integer LOCK_WAIT_TIMEOUT = 1200;
+    protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadurl";
+    protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository";
     protected static final List<CPU.CPUArch> DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList(
+            CPU.CPUArch.amd64,
             CPU.CPUArch.arm64
     );
 
@@ -820,6 +823,14 @@ public static String parseMetadataFile() {
             LOGGER.error(errMsg);
             throw new CloudRuntimeException(errMsg);
         }
+        Ini.Section defaultSection = ini.get("default");
+        boolean updateCustomDownloadRepository = false;
+        String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY);
+        String customDownloadRepository = System.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY);
+        if (StringUtils.isNotBlank(customDownloadRepository) && StringUtils.isNotBlank(defaultDownloadRepository)) {
+            LOGGER.debug("Updating custom download repository: {}", customDownloadRepository);
+            updateCustomDownloadRepository = true;
+        }
         for (Pair<Hypervisor.HypervisorType, CPU.CPUArch> hypervisorType : hypervisorList) {
             String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second());
             Ini.Section section = ini.get(key);
@@ -828,16 +839,21 @@ public static String parseMetadataFile() {
                         key, metadataFilePath);
                 continue;
             }
+            String url = section.get("downloadurl");
+            if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) {
+                url = url.replaceFirst(defaultDownloadRepository.trim(),
+                        customDownloadRepository.trim());
+                LOGGER.info("Updated download URL for {} to {}", key, url);
+            }
             NewTemplateMap.put(key, new MetadataTemplateDetails(
                     hypervisorType.first(),
                     section.get("templatename"),
                     section.get("filename"),
-                    section.get("downloadurl"),
+                    url,
                     section.get("checksum"),
                     hypervisorType.second(),
                     section.get("guestos")));
         }
-        Ini.Section defaultSection = ini.get("default");
         return defaultSection.get("version").trim();
     }
 

engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java

@@ -344,7 +344,7 @@ public void testIsTemplateFileChecksumDifferent_mismatch() {
     @Test(expected = CloudRuntimeException.class)
     public void testValidateTemplates_metadataTemplateFailure() {
         List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> list = new ArrayList<>();
-        list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64));
+        list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64));
         systemVmTemplateRegistration.validateTemplates(list);
     }
 

engine/schema/templateConfig.sh

@@ -27,6 +27,7 @@ function getTemplateVersion() {
   export CS_VERSION="${subversion1}"."${subversion2}"
   export CS_MINOR_VERSION="${minorversion}"
   export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}"
+  export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/${CS_VERSION}/"
 }
 
 function getGenericName() {
@@ -63,7 +64,7 @@ function getChecksum() {
 
 function createMetadataFile() {
   local fileData=$(cat $SOURCEFILE)
-  echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE
+  echo -e "["default"]\nversion = $VERSION.${securityversion}\ndownloadrepository = $CS_SYSTEMTEMPLATE_REPO\n" >> $METADATAFILE
   for template in "${templates[@]}"
   do
     section="${template%%:*}"
@@ -82,13 +83,21 @@ function createMetadataFile() {
 
 declare -a templates
 getTemplateVersion $1
-templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
-            "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2"
-            "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova"
-            "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2"
-            "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip"
-            "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2"
-            "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" )
+declare -A template_specs=(
+  [kvm-x86_64]="x86_64-kvm.qcow2.bz2"
+  [kvm-aarch64]="aarch64-kvm.qcow2.bz2"
+  [vmware]="x86_64-vmware.ova"
+  [xenserver]="x86_64-xen.vhd.bz2"
+  [hyperv4]="x86_64-hyperv.vhd.zip"
+  [lxc]="x86_64-kvm.qcow2.bz2"
+  [ovm3]="x86_64-ovm.raw.bz2"
+)
+
+templates=()
+for key in "${!template_specs[@]}"; do
+  url="${CS_SYSTEMTEMPLATE_REPO}/systemvmtemplate-$VERSION-${template_specs[$key]}"
+  templates+=("$key:$url")
+done
 
 PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/"
 mkdir -p $PARENTPATH

plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java

@@ -20,7 +20,6 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.file.Files;
@@ -51,6 +50,7 @@
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.extension.Extension;
 import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager;
+import org.apache.cloudstack.utils.ServerPropertiesUtil;
 import org.apache.cloudstack.utils.security.DigestHelper;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
@@ -80,7 +80,6 @@
 import com.cloud.serializer.GsonHelper;
 import com.cloud.utils.FileUtil;
 import com.cloud.utils.Pair;
-import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.component.PluggableService;
@@ -226,29 +225,6 @@ protected String getSanitizedJsonStringForLog(String json) {
         return json.replaceAll("(\"password\"\\s*:\\s*\")([^\"]*)(\")", "$1****$3");
     }
 
-    private String getServerProperty(String name) {
-        Properties props = propertiesRef.get();
-        if (props == null) {
-            File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE);
-            if (propsFile == null) {
-                logger.error("{} file not found", PROPERTIES_FILE);
-                return null;
-            }
-            Properties tempProps = new Properties();
-            try (FileInputStream is = new FileInputStream(propsFile)) {
-                tempProps.load(is);
-            } catch (IOException e) {
-                logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e);
-                return null;
-            }
-            if (!propertiesRef.compareAndSet(null, tempProps)) {
-                tempProps = propertiesRef.get();
-            }
-            props = tempProps;
-        }
-        return props.getProperty(name);
-    }
-
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         super.configure(name, params);
@@ -260,7 +236,7 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
     }
 
     private void initializeExtensionDirectories() {
-        String deploymentMode = getServerProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME);
+        String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME);
         if ("developer".equals(deploymentMode)) {
             extensionsDirectory = EXTENSIONS_DIRECTORY_DEV;
             extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV;

utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cloud.utils.PropertiesUtil;
+
+public class ServerPropertiesUtil {
+    private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class);
+    private static final String PROPERTIES_FILE = "server.properties";
+    private static final AtomicReference<Properties> propertiesRef = new AtomicReference<>();
+
+    public static String getProperty(String name) {
+        Properties props = propertiesRef.get();
+        if (props != null) {
+            return props.getProperty(name);
+        }
+        File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE);
+        if (propsFile == null) {
+            logger.error("{} file not found", PROPERTIES_FILE);
+            return null;
+        }
+        Properties tempProps = new Properties();
+        try (FileInputStream is = new FileInputStream(propsFile)) {
+            tempProps.load(is);
+        } catch (IOException e) {
+            logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e);
+            return null;
+        }
+        if (!propertiesRef.compareAndSet(null, tempProps)) {
+            tempProps = propertiesRef.get();
+        }
+        return tempProps.getProperty(name);
+    }
+}