diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 84c87ae2e442..6993f5dab793 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -36,6 +36,100 @@ from cloudutils.cloudException import CloudRuntimeException, CloudInternalExcept from cloudutils.globalEnv import globalEnv from cloudutils.serviceConfigServer import cloudManagementConfig from optparse import OptionParser +import urllib.request +import configparser +import hashlib + +SYSTEMVM_TEMPLATES_PATH = "/usr/share/cloudstack-management/templates/systemvm" +SYSTEMVM_TEMPLATES_METADATA_FILE = SYSTEMVM_TEMPLATES_PATH + "/metadata.ini" + +def verify_sha512_checksum(file_path, expected_checksum): + sha512 = hashlib.sha512() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha512.update(chunk) + return sha512.hexdigest().lower() == expected_checksum.lower() + except Exception as e: + print(f"Failed to verify checksum for {file_path}: {e}") + return False + +def download_file(url, dest_path, chunk_size=8 * 1024 * 1024): + """ + Downloads a file from the given URL to the specified destination path in chunks. + """ + try: + with urllib.request.urlopen(url) as response: + total_size = response.length if response.length else None + downloaded = 0 + try: + with open(dest_path, 'wb') as out_file: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + out_file.write(chunk) + downloaded += len(chunk) + if total_size: + print(f"Downloaded {downloaded / (1024 * 1024):.2f}MB of {total_size / (1024 * 1024):.2f}MB", end='\r') + except PermissionError as pe: + print(f"Permission denied: {dest_path}") + raise + print(f"\nDownloaded file from {url} to {dest_path}") + except Exception as e: + print(f"Failed to download file: {e}") + raise + +def download_template_if_needed(template, url, filename, checksum): + dest_path = os.path.join(SYSTEMVM_TEMPLATES_PATH, filename) + if os.path.exists(dest_path): + if checksum and verify_sha512_checksum(dest_path, checksum): + print(f"{template} System VM template already exists at {dest_path} with valid checksum, skipping download.") + return + else: + print(f"{template} System VM template at {dest_path} has invalid or missing checksum, re-downloading...") + else: + print(f"Downloading {template} System VM template from {url} to {dest_path}...") + try: + download_file(url, dest_path) + except Exception as e: + print(f"ERROR: Failed to download {template} System VM template: {e}") + +def collect_template_metadata(selected_templates, options): + template_metadata_list = [] + if not os.path.exists(SYSTEMVM_TEMPLATES_METADATA_FILE): + print(f"ERROR: System VM templates metadata file not found at {SYSTEMVM_TEMPLATES_METADATA_FILE}, cannot download templates.") + sys.exit(1) + config = configparser.ConfigParser() + config.read(SYSTEMVM_TEMPLATES_METADATA_FILE) + template_repo_url = None + if options.systemvm_templates_repository: + if "default" in config and "downloadrepository" in config["default"]: + template_repo_url = config["default"]["downloadrepository"].strip() + if not template_repo_url: + print("ERROR: downloadrepository value is empty in metadata file, cannot use --systemvm-template-repository option.") + sys.exit(1) + for template in selected_templates: + if template in config: + url = config[template].get("downloadurl") + filename = config[template].get("filename") + checksum = config[template].get("checksum") + if url and filename: + if template_repo_url: + url = url.replace(template_repo_url, options.systemvm_templates_repository) + template_metadata_list.append({ + "template": template, + "url": url, + "filename": filename, + "checksum": checksum + }) + else: + print(f"ERROR: URL or filename not found for {template} System VM template in metadata.") + sys.exit(1) + else: + print(f"ERROR: No metadata found for {template} System VM template.") + sys.exit(1) + return template_metadata_list if __name__ == '__main__': initLoging("@MSLOGDIR@/setupManagement.log") @@ -45,6 +139,16 @@ if __name__ == '__main__': parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server") parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it") parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration") + parser.add_option( + "--systemvm-templates", + dest="systemvm_templates", + help="Specify System VM templates to download: all, kvm-aarch64, kvm-x86_64, xenserver, vmware or comma-separated list of hypervisor combinations (e.g., kvm-x86_64,xenserver). Default is kvm-x86_64.", + ) + parser.add_option( + "--systemvm-templates-repository", + dest="systemvm_templates_repository", + help="Specify the URL to download System VM templates from." + ) (options, args) = parser.parse_args() if options.https: glbEnv.svrMode = "HttpsServer" @@ -53,6 +157,22 @@ if __name__ == '__main__': if options.nostart: glbEnv.noStart = True + available_templates = ["kvm-aarch64", "kvm-x86_64", "xenserver", "vmware"] + templates_arg = options.systemvm_templates + + selected_templates = ["kvm-x86_64"] + if templates_arg: + templates_list = [t.strip().lower() for t in templates_arg.split(",")] + if "all" in templates_list: + selected_templates = available_templates + else: + selected_templates = [t for t in templates_list if t in available_templates] + print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}") + + template_metadata_list = [] + if selected_templates: + template_metadata_list = collect_template_metadata(selected_templates, options) + glbEnv.mode = "Server" print("Starting to configure CloudStack Management Server:") @@ -68,9 +188,13 @@ if __name__ == '__main__': print("CloudStack Management Server setup is Done!") print("Please ensure ports 8080, 8250, 8443, and 9090 are opened and not firewalled for the management server and not in use by other processes on this host.") except (CloudRuntimeException, CloudInternalException) as e: + print(e) print("Try to restore your system:") try: syscfg.restore() except: pass + + for meta in template_metadata_list: + download_template_if_needed(meta["template"], meta["url"], meta["filename"], meta["checksum"]) diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 5958486b4dff..56d9925ab923 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -62,3 +62,8 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # Thread pool configuration #threads.min=10 #threads.max=500 + +# The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the +# `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL +# will be for download. +# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/4.20/ diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 9b1420f22c37..593478c3417d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -108,7 +108,10 @@ public class SystemVmTemplateRegistration { private static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; + protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadurl"; + protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository"; protected static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( + CPU.CPUArch.amd64, CPU.CPUArch.arm64 ); @@ -820,6 +823,14 @@ public static String parseMetadataFile() { LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } + Ini.Section defaultSection = ini.get("default"); + boolean updateCustomDownloadRepository = false; + String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); + String customDownloadRepository = System.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); + if (StringUtils.isNotBlank(customDownloadRepository) && StringUtils.isNotBlank(defaultDownloadRepository)) { + LOGGER.debug("Updating custom download repository: {}", customDownloadRepository); + updateCustomDownloadRepository = true; + } for (Pair hypervisorType : hypervisorList) { String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second()); Ini.Section section = ini.get(key); @@ -828,16 +839,21 @@ public static String parseMetadataFile() { key, metadataFilePath); continue; } + String url = section.get("downloadurl"); + if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { + url = url.replaceFirst(defaultDownloadRepository.trim(), + customDownloadRepository.trim()); + LOGGER.info("Updated download URL for {} to {}", key, url); + } NewTemplateMap.put(key, new MetadataTemplateDetails( hypervisorType.first(), section.get("templatename"), section.get("filename"), - section.get("downloadurl"), + url, section.get("checksum"), hypervisorType.second(), section.get("guestos"))); } - Ini.Section defaultSection = ini.get("default"); return defaultSection.get("version").trim(); } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index dceb8e07b07d..b9b0cbcf0011 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -344,7 +344,7 @@ public void testIsTemplateFileChecksumDifferent_mismatch() { @Test(expected = CloudRuntimeException.class) public void testValidateTemplates_metadataTemplateFailure() { List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); + list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64)); systemVmTemplateRegistration.validateTemplates(list); } diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index d6d1809c24d4..b899624d5557 100755 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -27,6 +27,7 @@ function getTemplateVersion() { export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" + export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/${CS_VERSION}/" } function getGenericName() { @@ -63,7 +64,7 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) - echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE + echo -e "["default"]\nversion = $VERSION.${securityversion}\ndownloadrepository = $CS_SYSTEMTEMPLATE_REPO\n" >> $METADATAFILE for template in "${templates[@]}" do section="${template%%:*}" @@ -82,13 +83,21 @@ function createMetadataFile() { declare -a templates getTemplateVersion $1 -templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2" - "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" - "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" - "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" - "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) +declare -A template_specs=( + [kvm-x86_64]="x86_64-kvm.qcow2.bz2" + [kvm-aarch64]="aarch64-kvm.qcow2.bz2" + [vmware]="x86_64-vmware.ova" + [xenserver]="x86_64-xen.vhd.bz2" + [hyperv4]="x86_64-hyperv.vhd.zip" + [lxc]="x86_64-kvm.qcow2.bz2" + [ovm3]="x86_64-ovm.raw.bz2" +) + +templates=() +for key in "${!template_specs[@]}"; do + url="${CS_SYSTEMTEMPLATE_REPO}/systemvmtemplate-$VERSION-${template_specs[$key]}" + templates+=("$key:$url") +done PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" mkdir -p $PARENTPATH diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java index 6cec5181de63..443e88230cd1 100644 --- a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -20,7 +20,6 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; @@ -51,6 +50,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.apache.cloudstack.utils.ServerPropertiesUtil; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -80,7 +80,6 @@ import com.cloud.serializer.GsonHelper; import com.cloud.utils.FileUtil; import com.cloud.utils.Pair; -import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -226,29 +225,6 @@ protected String getSanitizedJsonStringForLog(String json) { return json.replaceAll("(\"password\"\\s*:\\s*\")([^\"]*)(\")", "$1****$3"); } - private String getServerProperty(String name) { - Properties props = propertiesRef.get(); - if (props == null) { - File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); - if (propsFile == null) { - logger.error("{} file not found", PROPERTIES_FILE); - return null; - } - Properties tempProps = new Properties(); - try (FileInputStream is = new FileInputStream(propsFile)) { - tempProps.load(is); - } catch (IOException e) { - logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); - return null; - } - if (!propertiesRef.compareAndSet(null, tempProps)) { - tempProps = propertiesRef.get(); - } - props = tempProps; - } - return props.getProperty(name); - } - @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); @@ -260,7 +236,7 @@ public boolean configure(String name, Map params) throws Configu } private void initializeExtensionDirectories() { - String deploymentMode = getServerProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); + String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); if ("developer".equals(deploymentMode)) { extensionsDirectory = EXTENSIONS_DIRECTORY_DEV; extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV; diff --git a/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java new file mode 100644 index 000000000000..e1cb230cad11 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloud.utils.PropertiesUtil; + +public class ServerPropertiesUtil { + private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + private static final String PROPERTIES_FILE = "server.properties"; + private static final AtomicReference propertiesRef = new AtomicReference<>(); + + public static String getProperty(String name) { + Properties props = propertiesRef.get(); + if (props != null) { + return props.getProperty(name); + } + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + return tempProps.getProperty(name); + } +}