Merge branch 'main' into experi/tf-images

Author: sjpb

ansible/filter_plugins/utils.py

@@ -41,6 +41,9 @@ def to_ood_regex(items):
         eg {{ [compute-001, compute-002, control] | to_regex }} -> '(compute-\d+)|(control)'
     """
 
+    # NB: for python3.12+ the \d in this function & docstring
+    # need to be raw strings. See https://docs.python.org/3/reference/lexical_analysis.html
+
     # There's a python bug which means re.sub() can't use '\d' in the replacement so
     # have to do replacement in two stages:
     r = [re.sub(r"\d+", 'XBACKSLASHX', v) for v in items]

ansible/roles/dnf_repos/defaults/main.yml

@@ -17,7 +17,7 @@ dnf_repos_filenames:
 dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
 
 # epel installed separately
-dnf_repos_repolist:
+dnf_repos_default_repolist:
 - file: "{{ dnf_repos_version_filenames.baseos }}"
   name: baseos
   base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
@@ -30,6 +30,19 @@ dnf_repos_repolist:
 - file: "{{ dnf_repos_version_filenames.extras }}"
   name: extras
   base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
+- file: ceph
+  name: Ceph
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+
+dnf_repos_openhpc_repolist:
+- name: OpenHPC
+  file: OpenHPC
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+- name: OpenHPC-updates
+  file: OpenHPC
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+
+dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) }}"
 
 dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
 dnf_repos_epel_description: "epel"

ansible/roles/pulp_site/defaults/main.yml

@@ -22,6 +22,12 @@ pulp_site_rpm_info:
   subpath: "{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
 - name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major].timestamp }}"
   subpath: "{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ohpc-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ohpc-updates-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+- name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}"
+  subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
 
 pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"

dev/setup-env.sh

@@ -17,6 +17,7 @@ PYTHON_VERSION=""
 if [[ "$OS" == "ubuntu" && "$MAJOR_VERSION" == "22" ]]; then
     PYTHON_VERSION="/usr/bin/python3.10"
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "8" ]]; then
+    # python3.9+ doesn't have selinux bindings
     PYTHON_VERSION="/usr/bin/python3.8" # use `sudo yum install python38` on Rocky Linux 8 to install this
 elif [[ "$OS" == "rocky" && "$MAJOR_VERSION" == "9" ]]; then
     PYTHON_VERSION="/usr/bin/python3.9"

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,10 +1,10 @@
 {
     "cluster_image_names": {
         "RL8": {
-            "default": "openhpc-RL8-250102-1138-77cfc703"
+            "default": "openhpc-RL8-250107-1534-b03caaf3"
         },
         "RL9": {
-            "default": "openhpc-RL9-250102-1139-77cfc703"
+            "default": "openhpc-RL9-250107-1535-b03caaf3"
         }
     }
 }

environments/common/inventory/group_vars/all/defaults.yml

@@ -151,3 +151,24 @@ appliances_pulp_repos:
     '8':
       timestamp: 20241216T235733
       path: epel/8/Everything/x86_64
+  openhpc_base:
+    '8':
+      path: OpenHPC/2/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/EL_9
+      timestamp: 20241218T154614
+  openhpc_updates:
+    '8':
+      path: OpenHPC/2/updates/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/updates/EL_9
+      timestamp: 20241218T154614
+  ceph:
+    '8':
+      timestamp: 20231104T015751
+      path: centos/8-stream/storage/x86_64/ceph-quincy
+    '9':
+      timestamp: 20240923T233036
+      path: centos/9-stream/storage/x86_64/ceph-reef

environments/common/inventory/group_vars/all/openhpc.yml

@@ -39,6 +39,15 @@ openhpc_config_extra: {}
 openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_extra, list_merge='append') }}"
 openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}"
 
+openhpc_install_type: ohpc # 'ohpc' or 'generic', see https://github.com/stackhpc/ansible-slurm-appliance/pull/326
+
+# Empty repo lists from stackhpc.openhpc role defaults, as these repofiles are
+# now generated by dnf_repos to allow injecting Ark creds:
+ohpc_openhpc_repos:
+  "9": []
+  "8": []
+
+# overriding to ensure doesn't overwrite Ark epel repo
 ohpc_default_extra_repos:
-  "9": [] #overriding to ensure doesn't overwrite ark epel repo
+  "9": []
   "8": []

environments/common/inventory/group_vars/all/os-manila-mount.yml

@@ -0,0 +1,3 @@
+# Empty repo lists from stackhpc.ansible-role-os-manila-mount role defaults, as these repofiles are
+# now generated by dnf_repos to allow injecting Ark creds:
+os_manila_mount_ceph_rpm_repos: []

requirements.txt

@@ -1,4 +1,4 @@
-ansible==8.0.0
+ansible==6.7.0 # cloudalchemy.prometheus uses ansible.builtin.include, removed in ansible-core==2.16 => ansible==9
 openstacksdk
 python-openstackclient==6.6.1 # v7.0.0 has a bug re. rebuild
 python-manilaclient
@@ -9,4 +9,4 @@ cookiecutter
 selinux # this is a shim to avoid having to use --system-site-packages, you still need sudo yum install libselinux-python3
 netaddr
 matplotlib
-pulp-cli==0.29.2
+pulp-cli==0.23.2

requirements.yml

@@ -21,7 +21,7 @@ roles:
     version: v3.1.5
   - src: https://github.com/stackhpc/ansible-role-os-manila-mount.git
     name: stackhpc.os-manila-mount
-    version: v24.11.0 # Support ceph quincy for RL9
+    version: v25.1.1
 
 collections:
   - name: containers.podman
@@ -49,6 +49,10 @@ collections:
   - name: https://github.com/azimuth-cloud/ansible-collection-image-utils
     type: git
     version: 0.4.0
+  # stackhpc.pulp has pulp.squeezer as dependency, any version, but latest
+  # requires newer ansible than can install
+  - name: pulp.squeezer
+    version: 0.0.15
   - name: stackhpc.pulp
     version: 0.5.5
 ...