From 62188a1d279fbaf7f51de69f6f8a9622ed285379 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 3 Dec 2025 22:08:03 +0000 Subject: [PATCH 01/27] first try --- cfa/cloudops/_cloudclient.py | 16 ++++- cfa/cloudops/auth.py | 116 +++++++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index e2bf6df..5f948bb 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -89,19 +89,29 @@ class CloudClient: def __init__( self, + keyvault: str = None, dotenv_path: str = None, use_sp: bool = False, use_federated: bool = False, **kwargs, ): logger.debug("Initializing CloudClient.") + if keyvault is None and not os.path.exists(".env") and dotenv_path is None: + try: + keyvault = os.environ["AZURE_KEYVAULT_NAME"] + except KeyError: + logger.error("Keyvault information not found.") # authenticate to get credentials if not use_sp and not use_federated: - self.cred = EnvCredentialHandler(dotenv_path=dotenv_path, **kwargs) + self.cred = EnvCredentialHandler( + dotenv_path=dotenv_path, keyvault=keyvault, **kwargs + ) self.method = "env" - logger.info("Using environment-based credentials.") + logger.info("Using managed identity credentials.") elif use_federated: - self.cred = DefaultCredentialHandler(dotenv_path=dotenv_path, **kwargs) + self.cred = DefaultCredentialHandler( + dotenv_path=dotenv_path, keyvault=keyvault, **kwargs + ) self.method = "default" logger.info("Using default credentials.") else: diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 026fa52..e8657dd 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -533,7 +533,7 @@ class EnvCredentialHandler(CredentialHandler): >>> handler = EnvCredentialHandler(dotenv_path="/path/to/.env") """ - def __init__(self, dotenv_path: str = None, **kwargs) -> None: + def __init__(self, dotenv_path: str = None, keyvault: str = None, **kwargs) -> None: """Initialize the EnvCredentialHandler. Loads environment variables from .env file and populates credential attributes from them. @@ -541,10 +541,12 @@ def __init__(self, dotenv_path: str = None, **kwargs) -> None: Args: dotenv_path (str, optional): Path to .env file to load environment variables from. If None, uses default .env file discovery. + keyvault (str, optional): Name of the Azure Key Vault to use for secrets. **kwargs: Additional keyword arguments to override specific credential attributes. """ logger.debug("Initializing EnvCredentialHandler.") load_env_vars(dotenv_path=dotenv_path) + get_conf = partial(get_config_val, config_dict=kwargs, try_env=True) for key in self.__dataclass_fields__.keys(): @@ -556,7 +558,7 @@ def __init__(self, dotenv_path: str = None, **kwargs) -> None: self.__setattr__("azure_batch_location", d.default_azure_batch_location) -def load_env_vars(dotenv_path=None): +def load_env_vars(dotenv_path=None, keyvault_name: str = None): """Load environment variables and Azure subscription information. Loads variables from a .env file (if specified), retrieves Azure subscription @@ -564,21 +566,31 @@ def load_env_vars(dotenv_path=None): Args: dotenv_path: Path to .env file to load. If None, uses default .env file discovery. + keyvault_name: Name of the Azure Key Vault to use for secrets. Example: >>> load_env_vars() # Load from default .env >>> load_env_vars("/path/to/.env") # Load from specific file """ + # get ManagedIdentityCredential + mid_cred = ManagedIdentityCredential() + logger.debug("Loading environment variables.") load_dotenv(dotenv_path=dotenv_path, override=True) - # get ManagedIdentityCredential to pull SubscriptionClient - mid_cred = ManagedIdentityCredential() + sub_c = SubscriptionClient(mid_cred) # pull in account info and save to environment vars account_info = list(sub_c.subscriptions.list())[0] os.environ["AZURE_SUBSCRIPTION_ID"] = account_info.subscription_id os.environ["AZURE_TENANT_ID"] = account_info.tenant_id os.environ["AZURE_RESOURCE_GROUP_NAME"] = account_info.display_name + + # get Key Vault secrets + get_keyvault_vars( + keyvault_name=os.getenv("AZURE_KEYVAULT_NAME"), + credential=mid_cred, + ) + # save default values d.set_env_vars() @@ -591,6 +603,7 @@ def __init__( azure_client_id: str = None, azure_client_secret: str = None, dotenv_path: str = None, + keyvault: str = None, **kwargs, ): """Initialize a Service Principal Credential Handler. @@ -611,6 +624,7 @@ def __init__( attempt to load from AZURE_CLIENT_SECRET environment variable. dotenv_path: Path to .env file to load environment variables from. If None, uses default .env file discovery. + keyvault: Name of the Azure Key Vault to use for secrets. **kwargs: Additional keyword arguments to override specific credential attributes. Raises: @@ -681,6 +695,16 @@ def __init__( [x.lower() for x in mandatory_environment_variables], goal="service principal credentials", ) + sp_cred = ClientSecretCredential( + tenant_id=self.azure_tenant_id, + client_id=self.azure_client_id, + client_secret=self.azure_client_secret, + ) + # load keyvault secrets + get_keyvault_vars( + keyvault_name=keyvault, + credential=sp_cred, + ) d.set_env_vars() @@ -699,6 +723,7 @@ class DefaultCredentialHandler(CredentialHandler): def __init__( self, dotenv_path: str | None = None, + keyvault: str = None, **kwargs, ) -> None: """Initialize a Default Credential Handler. @@ -711,6 +736,7 @@ def __init__( Args: dotenv_path: Path to .env file to load environment variables from. If None, uses default .env file discovery. + keyvault: Name of the Azure Key Vault to use for secrets. **kwargs: Additional keyword arguments to override specific credential attributes. Raises: @@ -732,6 +758,13 @@ def __init__( ) d_cred = DefaultCredential() sub_c = SubscriptionClient(d_cred) + + # load keyvault secrets + get_keyvault_vars( + keyvault_name=keyvault, + credential=d_cred, + ) + # pull subscription id from env vars sub_id = os.getenv("AZURE_SUBSCRIPTION_ID", None) if sub_id is None: logger.error("AZURE_SUBSCRIPTION_ID not found in environment variables.") @@ -929,3 +962,78 @@ def get_compute_node_identity_reference( ch = EnvCredentialHandler() logger.debug("Retrieving compute_node_identity_reference from CredentialHandler.") return ch.compute_node_identity_reference + + +def get_secret_client(keyvault: str, credential: object) -> SecretClient: + """Get an Azure Key Vault SecretClient using a CredentialHandler. + + Args: + keyvault: Name of the Azure Key Vault to connect to. + credential: Credential handler for connecting and authenticating to Azure resources. + + Returns: + SecretClient: An authenticated SecretClient for the specified Key Vault. + + Example: + >>> handler = CredentialHandler() + >>> secret_client = get_secret_client("myvault", handler) + """ + logger.debug("Creating SecretClient for Azure Key Vault.") + vault_url = f"https://{keyvault}.{d.default_azure_keyvault_endpoint_subdomain}" + secret_client = SecretClient(vault_url=vault_url, credential=credential) + logger.debug("Created SecretClient for Azure Key Vault.") + return secret_client + + +def load_keyvault_vars( + secret_client: SecretClient, +): + """Load secrets from an Azure Key Vault into environment variables. + + Args: + secret_client: SecretClient for accessing the Azure Key Vault. + """ + kv_keys = [ + "azure_batch_account", + "azure_batch_location", + "azure_user_assigned_identity", + "azure_subnet_id", + "azure_client_id", + "azure_keyvault_sp_secret_id", + "azure_blob_storage_account", + "azure_container_registry_account", + ] + for key in kv_keys: + if key.upper() in os.environ: + logger.debug( + f"Environment variable '{key.upper()}' already set; skipping Key Vault load." + ) + continue + else: + try: + secret = secret_client.get_secret(key).value + os.environ[key.upper()] = secret + logger.debug( + f"Loaded secret '{key}' from Key Vault into environment variable." + ) + except Exception as e: + logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") + + +def get_keyvault_vars( + keyvault_name: str, + credential: object, +): + """Retrieve secrets from an Azure Key Vault and save to environment. + + Args: + keyvault_name: Name of the Azure Key Vault to connect to. + credential: Credential handler for connecting and authenticating to Azure resources. + """ + logger.debug("Getting SecretClient for Azure Key Vault.") + secret_client = get_secret_client( + keyvault=keyvault_name, + credential=credential, + ) + logger.debug("Loading Key Vault secrets into environment variables.") + load_keyvault_vars(secret_client) From 91d2b9aeb3dd6f2621c2a0c0faf2a2afce866797 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Thu, 4 Dec 2025 17:41:52 +0000 Subject: [PATCH 02/27] add a force_keyvault argument --- cfa/cloudops/_cloudclient.py | 23 +++++++++++++--- cfa/cloudops/auth.py | 51 +++++++++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index 5f948bb..5508a89 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -93,6 +93,7 @@ def __init__( dotenv_path: str = None, use_sp: bool = False, use_federated: bool = False, + force_keyvault: bool = False, **kwargs, ): logger.debug("Initializing CloudClient.") @@ -101,21 +102,37 @@ def __init__( keyvault = os.environ["AZURE_KEYVAULT_NAME"] except KeyError: logger.error("Keyvault information not found.") + if keyvault is None and force_keyvault: + logger.error( + "Keyvault information not found but force_keyvault set to True." + ) + raise ValueError("Keyvault information is required but not found.") # authenticate to get credentials if not use_sp and not use_federated: self.cred = EnvCredentialHandler( - dotenv_path=dotenv_path, keyvault=keyvault, **kwargs + dotenv_path=dotenv_path, + keyvault=keyvault, + force_keyvault=force_keyvault, + **kwargs, ) self.method = "env" logger.info("Using managed identity credentials.") elif use_federated: self.cred = DefaultCredentialHandler( - dotenv_path=dotenv_path, keyvault=keyvault, **kwargs + dotenv_path=dotenv_path, + keyvault=keyvault, + force_keyvault=force_keyvault, + **kwargs, ) self.method = "default" logger.info("Using default credentials.") else: - self.cred = SPCredentialHandler(dotenv_path=dotenv_path, **kwargs) + self.cred = SPCredentialHandler( + dotenv_path=dotenv_path, + keyvault=keyvault, + force_keyvault=force_keyvault, + **kwargs, + ) self.method = "sp" logger.info("Using service principal credentials.") # get clients diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index e8657dd..139c6f5 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -533,7 +533,13 @@ class EnvCredentialHandler(CredentialHandler): >>> handler = EnvCredentialHandler(dotenv_path="/path/to/.env") """ - def __init__(self, dotenv_path: str = None, keyvault: str = None, **kwargs) -> None: + def __init__( + self, + dotenv_path: str = None, + keyvault: str = None, + force_keyvault: bool = False, + **kwargs, + ) -> None: """Initialize the EnvCredentialHandler. Loads environment variables from .env file and populates credential attributes from them. @@ -545,7 +551,11 @@ def __init__(self, dotenv_path: str = None, keyvault: str = None, **kwargs) -> N **kwargs: Additional keyword arguments to override specific credential attributes. """ logger.debug("Initializing EnvCredentialHandler.") - load_env_vars(dotenv_path=dotenv_path) + load_env_vars( + dotenv_path=dotenv_path, + keyvault_name=keyvault, + force_keyvault=force_keyvault, + ) get_conf = partial(get_config_val, config_dict=kwargs, try_env=True) @@ -558,7 +568,9 @@ def __init__(self, dotenv_path: str = None, keyvault: str = None, **kwargs) -> N self.__setattr__("azure_batch_location", d.default_azure_batch_location) -def load_env_vars(dotenv_path=None, keyvault_name: str = None): +def load_env_vars( + dotenv_path=None, keyvault_name: str = None, force_keyvault: bool = False +): """Load environment variables and Azure subscription information. Loads variables from a .env file (if specified), retrieves Azure subscription @@ -567,6 +579,7 @@ def load_env_vars(dotenv_path=None, keyvault_name: str = None): Args: dotenv_path: Path to .env file to load. If None, uses default .env file discovery. keyvault_name: Name of the Azure Key Vault to use for secrets. + force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. Example: >>> load_env_vars() # Load from default .env @@ -587,8 +600,9 @@ def load_env_vars(dotenv_path=None, keyvault_name: str = None): # get Key Vault secrets get_keyvault_vars( - keyvault_name=os.getenv("AZURE_KEYVAULT_NAME"), + keyvault_name=keyvault_name, credential=mid_cred, + force_keyvault=force_keyvault, ) # save default values @@ -604,6 +618,7 @@ def __init__( azure_client_secret: str = None, dotenv_path: str = None, keyvault: str = None, + force_keyvault: bool = False, **kwargs, ): """Initialize a Service Principal Credential Handler. @@ -704,6 +719,7 @@ def __init__( get_keyvault_vars( keyvault_name=keyvault, credential=sp_cred, + force_keyvault=force_keyvault, ) d.set_env_vars() @@ -724,6 +740,7 @@ def __init__( self, dotenv_path: str | None = None, keyvault: str = None, + force_keyvault: bool = False, **kwargs, ) -> None: """Initialize a Default Credential Handler. @@ -763,6 +780,7 @@ def __init__( get_keyvault_vars( keyvault_name=keyvault, credential=d_cred, + force_keyvault=force_keyvault, ) # pull subscription id from env vars sub_id = os.getenv("AZURE_SUBSCRIPTION_ID", None) @@ -987,6 +1005,7 @@ def get_secret_client(keyvault: str, credential: object) -> SecretClient: def load_keyvault_vars( secret_client: SecretClient, + force_keyvault: bool = False, ): """Load secrets from an Azure Key Vault into environment variables. @@ -1004,12 +1023,10 @@ def load_keyvault_vars( "azure_container_registry_account", ] for key in kv_keys: - if key.upper() in os.environ: + if force_keyvault: logger.debug( - f"Environment variable '{key.upper()}' already set; skipping Key Vault load." + "Force Key Vault load enabled; loading secret regardless of existing environment variable." ) - continue - else: try: secret = secret_client.get_secret(key).value os.environ[key.upper()] = secret @@ -1018,11 +1035,27 @@ def load_keyvault_vars( ) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") + else: + if key.upper() in os.environ: + logger.debug( + f"Environment variable '{key.upper()}' already set; skipping Key Vault load." + ) + continue + else: + try: + secret = secret_client.get_secret(key).value + os.environ[key.upper()] = secret + logger.debug( + f"Loaded secret '{key}' from Key Vault into environment variable." + ) + except Exception as e: + logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") def get_keyvault_vars( keyvault_name: str, credential: object, + force_keyvault: bool = False, ): """Retrieve secrets from an Azure Key Vault and save to environment. @@ -1036,4 +1069,4 @@ def get_keyvault_vars( credential=credential, ) logger.debug("Loading Key Vault secrets into environment variables.") - load_keyvault_vars(secret_client) + load_keyvault_vars(secret_client, force_keyvault=force_keyvault) From 22a1255d662557a12c08c4def60f22d0b16c4c0b Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Thu, 4 Dec 2025 17:58:30 +0000 Subject: [PATCH 03/27] update doc strings --- cfa/cloudops/auth.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 139c6f5..22983ec 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -548,6 +548,7 @@ def __init__( dotenv_path (str, optional): Path to .env file to load environment variables from. If None, uses default .env file discovery. keyvault (str, optional): Name of the Azure Key Vault to use for secrets. + force_keyvault (bool, optional): If True, forces loading of Key Vault secrets even if they are already set in the environment. **kwargs: Additional keyword arguments to override specific credential attributes. """ logger.debug("Initializing EnvCredentialHandler.") @@ -640,6 +641,7 @@ def __init__( dotenv_path: Path to .env file to load environment variables from. If None, uses default .env file discovery. keyvault: Name of the Azure Key Vault to use for secrets. + force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. **kwargs: Additional keyword arguments to override specific credential attributes. Raises: @@ -754,6 +756,7 @@ def __init__( dotenv_path: Path to .env file to load environment variables from. If None, uses default .env file discovery. keyvault: Name of the Azure Key Vault to use for secrets. + force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. **kwargs: Additional keyword arguments to override specific credential attributes. Raises: @@ -1011,6 +1014,7 @@ def load_keyvault_vars( Args: secret_client: SecretClient for accessing the Azure Key Vault. + force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. """ kv_keys = [ "azure_batch_account", @@ -1062,7 +1066,11 @@ def get_keyvault_vars( Args: keyvault_name: Name of the Azure Key Vault to connect to. credential: Credential handler for connecting and authenticating to Azure resources. + force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. """ + if keyvault_name is None: + logger.debug("No Key Vault name provided; skipping Key Vault variable loading.") + return None logger.debug("Getting SecretClient for Azure Key Vault.") secret_client = get_secret_client( keyvault=keyvault_name, From 3ddb3175f59ca772444fb349787ccd3b519f04a6 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Thu, 4 Dec 2025 20:53:35 +0000 Subject: [PATCH 04/27] add kv function to cloudclient --- cfa/cloudops/_cloudclient.py | 27 ++++++++++++++++++++++++ docs/CloudClient/authentication.md | 33 ++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index 5508a89..b7bdf2e 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -3,6 +3,7 @@ import logging import os from graphlib import CycleError, TopologicalSorter +from typing import Optional import networkx as nx import pandas as pd @@ -13,6 +14,7 @@ OnAllTasksComplete, OnTaskFailure, ) +from azure.keyvault.secrets import SecretClient # from azure.batch.models import TaskAddParameter from azure.mgmt.batch import models @@ -1674,6 +1676,7 @@ def async_upload_folder( location_in_blob="project") Note: + The blob container must exist before uploading. Directory structure is preserved in the container. Use filtering options to avoid uploading unnecessary files like temporary files or build artifacts. @@ -2150,3 +2153,27 @@ def run_dag(self, *args: batch_helpers.Task, job_name: str, **kwargs): dlist.append(str(dp)) task_df.at[i, "deps"] = dlist logger.info(f"Completed DAG run for job '{job_name}'.") + + def get_kv_secret(self, secret_name: str, keyvault: str) -> Optional[str]: + """Retrieve a secret from Azure Key Vault. + + Args: + secret_name (str): The name of the secret to retrieve. + keyvault (str): The name of the Key Vault. + + Returns: + Optional[str]: The value of the secret, or None if not found. + """ + try: + secret_client = SecretClient( + vault_url=f"https://{keyvault}.vault.azure.net/", + credential=self.cred, + ) + secret = secret_client.get_secret(secret_name) + return secret.value + except Exception as e: + logger.error( + f"Failed to retrieve secret '{secret_name}' from Key Vault '{keyvault}': {e}" + ) + print(f"Error retrieving secret '{secret_name}': {e}") + return None diff --git a/docs/CloudClient/authentication.md b/docs/CloudClient/authentication.md index 3678a83..43c58f8 100644 --- a/docs/CloudClient/authentication.md +++ b/docs/CloudClient/authentication.md @@ -1,6 +1,6 @@ # Authentication with `cfa.cloudops.CloudClient` -Authentication with the `CloudClient` class is meant to be user-friendly while maintaining flexibility. There are three different ways to authenticate to the Azure environment, all of which center around environment variables for Azure account information. These environment variables can be pulled from the local environment or instantiated from a .env file specified during the `CloudClient` instantiation. +Authentication with the `CloudClient` class is meant to be user-friendly while maintaining flexibility. There are three different ways to authenticate to the Azure environment, all of which center around either a Key Vault or environment variables for Azure account information. A key vault name can be provided to pull necessary values for instantiating the CloudClient. Or these environment variables can be pulled from the local environment or instantiated from a .env file specified during the `CloudClient` instantiation. The three authentication methods available are: @@ -8,9 +8,38 @@ The three authentication methods available are: - Service Principal credential - Federated Token credential +## Using Key Vault Setup + +when the `CloudClient` class gets instantiated, one way it attempts to get one of the three credentials listed above is by pulling values from the specified `keyvault`. The Key Vault to be used by CFA individuals can be found in the documentation [here](https://github.com/cdcent/cfa-cloudops-example). This will then pull the following values from the Key Vault: + +- azure_batch_account +- azure_batch_location +- azure_user_assigned_identity +- azure_subnet_id +- azure_client_id +- azure_keyvault_sp_secret_id +- azure_blob_storage_account +- azure_container_registry_account + +If the Key Vault is setup with these keys/values (the correct CFA key vault is), then no .env file is necessary. If a .env is still provided, then values from the .env will be used over what is stored in the key vault. If you desire to use values in the keyvault over the .env, provide the flag `force_keyvault=True` when instantiating the `CloudClient`. Note that if you are using a service principal then "AZURE_TENANT_ID","AZURE_SUBSCRIPTION_ID", "AZURE_CLIENT_ID", and "AZURE_CLIENT_SECRET" need to be in the .env file, saved as local environment variables, or passed to the `CloudClient`. + +For ease of use, you can also set AZURE_KEYVAULT_NAME as a global environment variable in your development workspace so that it will be passed to the `CloudClient` and eliminate the need for any parameters when instantiating the CloudClient. + +For example, the following way pulls values from our Key Vault called 'my-key-vault'. + +```python3 +client = CloudClient(keyvault = "my-key-vault") +``` + +If we want to force the use of Key Vault values, the following should be run: + +```python3 +client = CloudClient(keyvault = "my-key-vault", force_keyvault = True) +``` + ## Environment Variable Setup -When the `CloudClient` class gets instantiated, it attempts to get one of the three credentials listed above based on environment variables. These environment variables can be stored locally on your system before calling out to the `CloudClient` class. A potentially easier way is to store the required variables is in a .env file. This allows for easier changing of variables or sharing between individuals. +When the `CloudClient` class gets instantiated, the other way it attempts to get one of the three credentials listed above is based on environment variables. These environment variables can be stored locally on your system before calling out to the `CloudClient` class. A potentially easier way is to store the required variables is in a .env file. This allows for easier changing of variables or sharing between individuals. The path to the .env file can be provided via the `dotenv_path` parameter when calling `CloudClient()`. By default, it looks for a file called `.env`. If the name of the file is anything else, it should be passed to `dotenv_path`. For example, instantiating the client in the following ways would be identical: ```python From b6be2e0a786ee92e5031a55e93f21b446c05df7b Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Thu, 4 Dec 2025 21:23:01 +0000 Subject: [PATCH 05/27] fix credentials --- cfa/cloudops/_cloudclient.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index b7bdf2e..c707a88 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -2164,10 +2164,16 @@ def get_kv_secret(self, secret_name: str, keyvault: str) -> Optional[str]: Returns: Optional[str]: The value of the secret, or None if not found. """ + if self.method == "env": + cred = self.cred.user_credential + elif self.method == "default": + cred = self.cred.client_secret_sp_credential + else: + cred = self.cred.client_secret_credential try: secret_client = SecretClient( vault_url=f"https://{keyvault}.vault.azure.net/", - credential=self.cred, + credential=cred, ) secret = secret_client.get_secret(secret_name) return secret.value From dce83366653bf7b2b9b6aaa7108cefba9f7b73b5 Mon Sep 17 00:00:00 2001 From: Ryan Raasch <150935395+ryanraaschCDC@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:16:03 +0000 Subject: [PATCH 06/27] Update docs/CloudClient/authentication.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/CloudClient/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CloudClient/authentication.md b/docs/CloudClient/authentication.md index 43c58f8..11dfc97 100644 --- a/docs/CloudClient/authentication.md +++ b/docs/CloudClient/authentication.md @@ -10,7 +10,7 @@ The three authentication methods available are: ## Using Key Vault Setup -when the `CloudClient` class gets instantiated, one way it attempts to get one of the three credentials listed above is by pulling values from the specified `keyvault`. The Key Vault to be used by CFA individuals can be found in the documentation [here](https://github.com/cdcent/cfa-cloudops-example). This will then pull the following values from the Key Vault: +When the `CloudClient` class gets instantiated, one way it attempts to get one of the three credentials listed above is by pulling values from the specified `keyvault`. The Key Vault to be used by CFA individuals can be found in the documentation [here](https://github.com/cdcent/cfa-cloudops-example). This will then pull the following values from the Key Vault: - azure_batch_account - azure_batch_location From b01726143b17597c6471856de0b042184aea8a43 Mon Sep 17 00:00:00 2001 From: Ryan Raasch <150935395+ryanraaschCDC@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:16:35 +0000 Subject: [PATCH 07/27] Update cfa/cloudops/_cloudclient.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cfa/cloudops/_cloudclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index c707a88..a639618 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -2167,7 +2167,7 @@ def get_kv_secret(self, secret_name: str, keyvault: str) -> Optional[str]: if self.method == "env": cred = self.cred.user_credential elif self.method == "default": - cred = self.cred.client_secret_sp_credential + cred = self.cred.user_credential else: cred = self.cred.client_secret_credential try: From a134588529cbee86fa746c3e195a493ca02d0558 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Fri, 5 Dec 2025 17:53:44 +0000 Subject: [PATCH 08/27] updated documentation --- docs/CloudClient/authentication.md | 5 +- .../cloudclient_walkthrough.ipynb | 46 ++++++++++++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/CloudClient/authentication.md b/docs/CloudClient/authentication.md index 11dfc97..513a1a0 100644 --- a/docs/CloudClient/authentication.md +++ b/docs/CloudClient/authentication.md @@ -52,8 +52,7 @@ If the .env file is called "my_azure.env" then the following should be run: client = CloudClient(dotenv_path = "my_azure.env") ``` -During instantiation of the `CloudClient`, the variables from the .env file get added to the local environment variables, overriding any variables with the same name. Then all the environment variables from the local environment are used to create a cre -dential. +During instantiation of the `CloudClient`, the variables from the .env file get added to the local environment variables, overriding any variables with the same name. Then all the environment variables from the local environment are used to create a credential. An example .env file can be found [here](../files/sample.env). @@ -64,7 +63,7 @@ An example .env file can be found [here](../files/sample.env). The default method for authenticating to the Azure environment via the `CloudClient` is a Managed Identity. Data Scientists at CFA should already have identities associated with Azure in their development environment (VAP). Because of this, we can reduce the number of inputs to authenticate with Azure because your machine is already approved. This is the encouraged method when possible. When this method is used, we are able to pull in AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, and AZURE_RESOURCE_GROUP_NAME from the linked subscription. Therefore, these values do not need to exist in the local environment or .env file. To instantiate a `CloudClient` object using a Managed Identity credential, no additional arguments need to be passed in, except from `dotenv_path` if needed. For example: -```python3 +```python client = CloudClient() ``` diff --git a/docs/examples/getting_started/cloudclient_walkthrough.ipynb b/docs/examples/getting_started/cloudclient_walkthrough.ipynb index 476f8bb..dea8f9f 100644 --- a/docs/examples/getting_started/cloudclient_walkthrough.ipynb +++ b/docs/examples/getting_started/cloudclient_walkthrough.ipynb @@ -38,7 +38,7 @@ "id": "b60f671d", "metadata": {}, "source": [ - "The initialization below is the simplest way to create and instance of the `CloudClient` class. It will use environment variables or values stored in a .env file to authenticate, like the .env file stored [here](../../files/sample.env), and a managed identity credential based on your local working environment. The .env file should be stored at the same level in the directory in which you're working." + "The initialization below is the simplest way to create and instance of the `CloudClient` class. If a variable called AZURE_KEYVAULT_NAME is saved to your environment, the `CloudClient` will initialize based on some Azure values stored in the Key Vault. Otherwise it will use environment variables or values stored in a .env file to authenticate, like the .env file stored [here](../../files/sample.env), and a managed identity credential based on your local working environment. The .env file should be stored at the same level in the directory in which you're working." ] }, { @@ -56,6 +56,50 @@ "cc = CloudClient()" ] }, + { + "cell_type": "markdown", + "id": "a2643149", + "metadata": {}, + "source": [ + "We could also specify the Key Vault directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c2f7e39", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "#cc = CloudClient(keyvault = 'my-key-vault')" + ] + }, + { + "cell_type": "markdown", + "id": "9d6d3c1d", + "metadata": {}, + "source": [ + "If we want to ensure we use values from the Key Vault even though they might exist in the .env file, we can specify this by setiing `force_keyvault=True`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f33ea7d0", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "# cc = CloudClient(keyvault = 'my-key-vault', force_keyvault = True)" + ] + }, { "cell_type": "markdown", "id": "8a3e55aa", From 670ad88a0cd608a7a2442f3c8142bdb0cdf70f37 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Mon, 8 Dec 2025 19:50:49 +0000 Subject: [PATCH 09/27] change keys to match key vault --- cfa/cloudops/auth.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 22983ec..eef1068 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -1017,14 +1017,14 @@ def load_keyvault_vars( force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. """ kv_keys = [ - "azure_batch_account", - "azure_batch_location", - "azure_user_assigned_identity", - "azure_subnet_id", - "azure_client_id", - "azure_keyvault_sp_secret_id", - "azure_blob_storage_account", - "azure_container_registry_account", + "AZURE_BATCH_ACCOUNT", + "AZURE_KEYVAULT_LOCATION", + "AZURE_USER_ASSIGNED_IDENTITY", + "AZURE_SUBNET_ID", + "AZURE_CLIENT_ID", + "AZURE_KEYVAULT_SP_SECRET_ID", + "AZURE_BLOB_STORAGE_ACCOUNT", + "AZURE_CONTAINER_REGISTRY_ACCOUNT", ] for key in kv_keys: if force_keyvault: @@ -1032,23 +1032,23 @@ def load_keyvault_vars( "Force Key Vault load enabled; loading secret regardless of existing environment variable." ) try: - secret = secret_client.get_secret(key).value - os.environ[key.upper()] = secret + secret = secret_client.get_secret(key.replace("_", "-")).value + os.environ[key] = secret logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") else: - if key.upper() in os.environ: + if key in os.environ: logger.debug( - f"Environment variable '{key.upper()}' already set; skipping Key Vault load." + f"Environment variable '{key}' already set; skipping Key Vault load." ) continue else: try: secret = secret_client.get_secret(key).value - os.environ[key.upper()] = secret + os.environ[key] = secret logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) From 8beba309052e170ae3b3ea454871e43900990ffb Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 9 Dec 2025 19:50:26 +0000 Subject: [PATCH 10/27] add more logic --- cfa/cloudops/auth.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index eef1068..152f844 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -600,11 +600,12 @@ def load_env_vars( os.environ["AZURE_RESOURCE_GROUP_NAME"] = account_info.display_name # get Key Vault secrets - get_keyvault_vars( - keyvault_name=keyvault_name, - credential=mid_cred, - force_keyvault=force_keyvault, - ) + if keyvault_name is not None: + get_keyvault_vars( + keyvault_name=keyvault_name, + credential=mid_cred, + force_keyvault=force_keyvault, + ) # save default values d.set_env_vars() @@ -718,11 +719,12 @@ def __init__( client_secret=self.azure_client_secret, ) # load keyvault secrets - get_keyvault_vars( - keyvault_name=keyvault, - credential=sp_cred, - force_keyvault=force_keyvault, - ) + if keyvault is not None: + get_keyvault_vars( + keyvault_name=keyvault, + credential=sp_cred, + force_keyvault=force_keyvault, + ) d.set_env_vars() @@ -780,11 +782,12 @@ def __init__( sub_c = SubscriptionClient(d_cred) # load keyvault secrets - get_keyvault_vars( - keyvault_name=keyvault, - credential=d_cred, - force_keyvault=force_keyvault, - ) + if keyvault is not None: + get_keyvault_vars( + keyvault_name=keyvault, + credential=d_cred, + force_keyvault=force_keyvault, + ) # pull subscription id from env vars sub_id = os.getenv("AZURE_SUBSCRIPTION_ID", None) if sub_id is None: From 6aa4d491cc949f3f0048bf22aee0df3172a1dfc4 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 10 Dec 2025 15:58:56 +0000 Subject: [PATCH 11/27] add a delete env --- cfa/cloudops/auth.py | 20 ++++++++++---------- cfa/cloudops/defaults.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 152f844..8eca522 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -589,6 +589,10 @@ def load_env_vars( # get ManagedIdentityCredential mid_cred = ManagedIdentityCredential() + # delete existing kv keys + for key in d.default_kv_keys: + del os.environ[key] + logger.debug("Loading environment variables.") load_dotenv(dotenv_path=dotenv_path, override=True) @@ -599,6 +603,10 @@ def load_env_vars( os.environ["AZURE_TENANT_ID"] = account_info.tenant_id os.environ["AZURE_RESOURCE_GROUP_NAME"] = account_info.display_name + # delete existing kv keys + for key in d.default_kv_keys: + del os.environ[key] + # get Key Vault secrets if keyvault_name is not None: get_keyvault_vars( @@ -1019,16 +1027,8 @@ def load_keyvault_vars( secret_client: SecretClient for accessing the Azure Key Vault. force_keyvault: If True, forces loading of Key Vault secrets even if they are already set in the environment. """ - kv_keys = [ - "AZURE_BATCH_ACCOUNT", - "AZURE_KEYVAULT_LOCATION", - "AZURE_USER_ASSIGNED_IDENTITY", - "AZURE_SUBNET_ID", - "AZURE_CLIENT_ID", - "AZURE_KEYVAULT_SP_SECRET_ID", - "AZURE_BLOB_STORAGE_ACCOUNT", - "AZURE_CONTAINER_REGISTRY_ACCOUNT", - ] + kv_keys = d.default_kv_keys + for key in kv_keys: if force_keyvault: logger.debug( diff --git a/cfa/cloudops/defaults.py b/cfa/cloudops/defaults.py index 8558421..2e867e2 100644 --- a/cfa/cloudops/defaults.py +++ b/cfa/cloudops/defaults.py @@ -137,6 +137,17 @@ def remaining_task_autoscale_formula( ), ) +default_kv_keys = [ + "AZURE_BATCH_ACCOUNT", + "AZURE_KEYVAULT_LOCATION", + "AZURE_USER_ASSIGNED_IDENTITY", + "AZURE_SUBNET_ID", + "AZURE_CLIENT_ID", + "AZURE_KEYVAULT_SP_SECRET_ID", + "AZURE_BLOB_STORAGE_ACCOUNT", + "AZURE_CONTAINER_REGISTRY_ACCOUNT", +] + def set_env_vars(): """Set default Azure environment variables. From 8fa162d5c37ddeaaf8b7993114e713492707a837 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 10 Dec 2025 16:13:30 +0000 Subject: [PATCH 12/27] add check if env var exists before del --- cfa/cloudops/auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 8eca522..5b175a3 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -591,7 +591,8 @@ def load_env_vars( # delete existing kv keys for key in d.default_kv_keys: - del os.environ[key] + if key in os.environ: + del os.environ[key] logger.debug("Loading environment variables.") load_dotenv(dotenv_path=dotenv_path, override=True) From 116db742ee368654c565a28176aa5d6d62b8d208 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 10 Dec 2025 16:22:50 +0000 Subject: [PATCH 13/27] remove extra del loop --- cfa/cloudops/auth.py | 4 ---- cfa/cloudops/defaults.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 5b175a3..93627a3 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -604,10 +604,6 @@ def load_env_vars( os.environ["AZURE_TENANT_ID"] = account_info.tenant_id os.environ["AZURE_RESOURCE_GROUP_NAME"] = account_info.display_name - # delete existing kv keys - for key in d.default_kv_keys: - del os.environ[key] - # get Key Vault secrets if keyvault_name is not None: get_keyvault_vars( diff --git a/cfa/cloudops/defaults.py b/cfa/cloudops/defaults.py index 2e867e2..5ba2dfb 100644 --- a/cfa/cloudops/defaults.py +++ b/cfa/cloudops/defaults.py @@ -139,7 +139,7 @@ def remaining_task_autoscale_formula( default_kv_keys = [ "AZURE_BATCH_ACCOUNT", - "AZURE_KEYVAULT_LOCATION", + "AZURE_BATCH_LOCATION", "AZURE_USER_ASSIGNED_IDENTITY", "AZURE_SUBNET_ID", "AZURE_CLIENT_ID", From 5f391a916572e830851420658f795b27659c656c Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 10 Dec 2025 19:30:27 +0000 Subject: [PATCH 14/27] remove del --- cfa/cloudops/auth.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 93627a3..9a3c053 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -589,11 +589,6 @@ def load_env_vars( # get ManagedIdentityCredential mid_cred = ManagedIdentityCredential() - # delete existing kv keys - for key in d.default_kv_keys: - if key in os.environ: - del os.environ[key] - logger.debug("Loading environment variables.") load_dotenv(dotenv_path=dotenv_path, override=True) From 102aebfd5b507fe8d3b046320648632260505c03 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 16 Dec 2025 15:21:57 +0000 Subject: [PATCH 15/27] add print statements for debugging --- cfa/cloudops/_cloudclient.py | 2 ++ cfa/cloudops/auth.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index a639618..6a822ac 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -99,6 +99,8 @@ def __init__( **kwargs, ): logger.debug("Initializing CloudClient.") + if keyvault: + print(keyvault) if keyvault is None and not os.path.exists(".env") and dotenv_path is None: try: keyvault = os.environ["AZURE_KEYVAULT_NAME"] diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 9a3c053..bddb80e 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -774,6 +774,7 @@ def __init__( """ logger.debug("Initializing DefaultCredentialHandler.") logger.debug("Loading environment variables.") + print(keyvault) load_dotenv(dotenv_path=dotenv_path) logger.debug( "Retrieving Azure subscription information using DefaultCredential." @@ -1022,6 +1023,7 @@ def load_keyvault_vars( kv_keys = d.default_kv_keys for key in kv_keys: + print(key) if force_keyvault: logger.debug( "Force Key Vault load enabled; loading secret regardless of existing environment variable." @@ -1029,6 +1031,7 @@ def load_keyvault_vars( try: secret = secret_client.get_secret(key.replace("_", "-")).value os.environ[key] = secret + print(secret[:2]) logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) @@ -1072,4 +1075,5 @@ def get_keyvault_vars( credential=credential, ) logger.debug("Loading Key Vault secrets into environment variables.") + print("loading keyvault vars") load_keyvault_vars(secret_client, force_keyvault=force_keyvault) From d7e7a917793430005e0711810e6f7cd46c46772a Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 16 Dec 2025 15:26:57 +0000 Subject: [PATCH 16/27] pull kv from env again --- cfa/cloudops/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index bddb80e..5d28a09 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -783,6 +783,11 @@ def __init__( sub_c = SubscriptionClient(d_cred) # load keyvault secrets + if keyvault is None: + try: + keyvault = os.environ["AZURE_KEYVAULT_NAME"] + except KeyError: + keyvault = None if keyvault is not None: get_keyvault_vars( keyvault_name=keyvault, From 11b86f3764864d65bb497ad689e15d3f7dc47315 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 16 Dec 2025 19:29:36 +0000 Subject: [PATCH 17/27] more prints --- cfa/cloudops/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 5d28a09..90fb404 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -1033,6 +1033,7 @@ def load_keyvault_vars( logger.debug( "Force Key Vault load enabled; loading secret regardless of existing environment variable." ) + print("force keyvault") try: secret = secret_client.get_secret(key.replace("_", "-")).value os.environ[key] = secret @@ -1042,11 +1043,13 @@ def load_keyvault_vars( ) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") + print("Error loading secret") else: if key in os.environ: logger.debug( f"Environment variable '{key}' already set; skipping Key Vault load." ) + print("Environment variable already set") continue else: try: @@ -1055,8 +1058,10 @@ def load_keyvault_vars( logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) + print(secret[:2]) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") + print("Error loading secret") def get_keyvault_vars( From 6c8d2b590c222988b19af2c051f4946ec3301eaa Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 16 Dec 2025 20:30:23 +0000 Subject: [PATCH 18/27] more prints and change where pulling sub client from --- cfa/cloudops/auth.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 90fb404..4c13125 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -774,27 +774,29 @@ def __init__( """ logger.debug("Initializing DefaultCredentialHandler.") logger.debug("Loading environment variables.") - print(keyvault) + print("Using keyvault:", keyvault) load_dotenv(dotenv_path=dotenv_path) logger.debug( "Retrieving Azure subscription information using DefaultCredential." ) d_cred = DefaultCredential() - sub_c = SubscriptionClient(d_cred) # load keyvault secrets if keyvault is None: + print("keyvault is None") try: keyvault = os.environ["AZURE_KEYVAULT_NAME"] except KeyError: keyvault = None if keyvault is not None: + print("keyvault is not None") get_keyvault_vars( keyvault_name=keyvault, credential=d_cred, force_keyvault=force_keyvault, ) # pull subscription id from env vars + sub_c = SubscriptionClient(d_cred) sub_id = os.getenv("AZURE_SUBSCRIPTION_ID", None) if sub_id is None: logger.error("AZURE_SUBSCRIPTION_ID not found in environment variables.") @@ -1061,7 +1063,7 @@ def load_keyvault_vars( print(secret[:2]) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") - print("Error loading secret") + print(f"Error loading secret: {e}") def get_keyvault_vars( From 770b9c0b04cbcc3f210aa543ab90a56827dc0800 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 17 Dec 2025 19:11:31 +0000 Subject: [PATCH 19/27] more print --- cfa/cloudops/auth.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 4c13125..40cc31c 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -796,7 +796,13 @@ def __init__( force_keyvault=force_keyvault, ) # pull subscription id from env vars - sub_c = SubscriptionClient(d_cred) + print("Attempting to get subscription client") + try: + sub_c = SubscriptionClient(d_cred) + print + except Exception as e: + logger.error(f"Failed to create SubscriptionClient: {e}") + raise sub_id = os.getenv("AZURE_SUBSCRIPTION_ID", None) if sub_id is None: logger.error("AZURE_SUBSCRIPTION_ID not found in environment variables.") @@ -804,11 +810,13 @@ def __init__( subscription = [ sub for sub in sub_c.subscriptions.list() if sub.subscription_id == sub_id ] + print("Got subscription info: ", subscription) # pull info if sub exists logger.debug("Pulling subscription information.") if subscription: subscription = subscription[0] os.environ["AZURE_RESOURCE_GROUP_NAME"] = subscription.display_name + print("Set AZURE_RESOURCE_GROUP_NAME:", subscription.display_name) logger.debug("Set AZURE_RESOURCE_GROUP_NAME from subscription information.") else: logger.error( From 1546de5a0fa8da97b8f78a6bbf2de0fb8a9f76a5 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Wed, 17 Dec 2025 20:11:10 +0000 Subject: [PATCH 20/27] print exceptions --- cfa/cloudops/auth.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 40cc31c..4b0b92a 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -1053,7 +1053,7 @@ def load_keyvault_vars( ) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") - print("Error loading secret") + print("Error loading secret: ", e) else: if key in os.environ: logger.debug( @@ -1090,10 +1090,15 @@ def get_keyvault_vars( logger.debug("No Key Vault name provided; skipping Key Vault variable loading.") return None logger.debug("Getting SecretClient for Azure Key Vault.") - secret_client = get_secret_client( - keyvault=keyvault_name, - credential=credential, - ) + print("Getting secret client") + try: + secret_client = get_secret_client( + keyvault=keyvault_name, + credential=credential, + ) + except Exception as e: + logger.error(f"Failed to get SecretClient: {e}") + raise logger.debug("Loading Key Vault secrets into environment variables.") print("loading keyvault vars") load_keyvault_vars(secret_client, force_keyvault=force_keyvault) From 3cf6ef42ca11cf6bdb3ff00a41cc4b1ff6a0f76e Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Thu, 18 Dec 2025 20:35:25 +0000 Subject: [PATCH 21/27] fix formatting of key name --- cfa/cloudops/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 4b0b92a..1da14e2 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -1063,7 +1063,7 @@ def load_keyvault_vars( continue else: try: - secret = secret_client.get_secret(key).value + secret = secret_client.get_secret(key.replace("_", "-")).value os.environ[key] = secret logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." From e6d3a87a34058a6b52a8f768c024486e6aea1bd9 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Mon, 5 Jan 2026 19:55:52 +0000 Subject: [PATCH 22/27] remove extra prints --- cfa/cloudops/auth.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 1da14e2..15b3e6f 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -774,7 +774,6 @@ def __init__( """ logger.debug("Initializing DefaultCredentialHandler.") logger.debug("Loading environment variables.") - print("Using keyvault:", keyvault) load_dotenv(dotenv_path=dotenv_path) logger.debug( "Retrieving Azure subscription information using DefaultCredential." @@ -783,23 +782,19 @@ def __init__( # load keyvault secrets if keyvault is None: - print("keyvault is None") try: keyvault = os.environ["AZURE_KEYVAULT_NAME"] except KeyError: keyvault = None if keyvault is not None: - print("keyvault is not None") get_keyvault_vars( keyvault_name=keyvault, credential=d_cred, force_keyvault=force_keyvault, ) - # pull subscription id from env vars - print("Attempting to get subscription client") + try: sub_c = SubscriptionClient(d_cred) - print except Exception as e: logger.error(f"Failed to create SubscriptionClient: {e}") raise @@ -810,13 +805,11 @@ def __init__( subscription = [ sub for sub in sub_c.subscriptions.list() if sub.subscription_id == sub_id ] - print("Got subscription info: ", subscription) # pull info if sub exists logger.debug("Pulling subscription information.") if subscription: subscription = subscription[0] os.environ["AZURE_RESOURCE_GROUP_NAME"] = subscription.display_name - print("Set AZURE_RESOURCE_GROUP_NAME:", subscription.display_name) logger.debug("Set AZURE_RESOURCE_GROUP_NAME from subscription information.") else: logger.error( @@ -1038,16 +1031,13 @@ def load_keyvault_vars( kv_keys = d.default_kv_keys for key in kv_keys: - print(key) if force_keyvault: logger.debug( "Force Key Vault load enabled; loading secret regardless of existing environment variable." ) - print("force keyvault") try: secret = secret_client.get_secret(key.replace("_", "-")).value os.environ[key] = secret - print(secret[:2]) logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) @@ -1059,7 +1049,6 @@ def load_keyvault_vars( logger.debug( f"Environment variable '{key}' already set; skipping Key Vault load." ) - print("Environment variable already set") continue else: try: @@ -1068,7 +1057,6 @@ def load_keyvault_vars( logger.debug( f"Loaded secret '{key}' from Key Vault into environment variable." ) - print(secret[:2]) except Exception as e: logger.warning(f"Could not load secret '{key}' from Key Vault: {e}") print(f"Error loading secret: {e}") @@ -1090,7 +1078,6 @@ def get_keyvault_vars( logger.debug("No Key Vault name provided; skipping Key Vault variable loading.") return None logger.debug("Getting SecretClient for Azure Key Vault.") - print("Getting secret client") try: secret_client = get_secret_client( keyvault=keyvault_name, @@ -1100,5 +1087,4 @@ def get_keyvault_vars( logger.error(f"Failed to get SecretClient: {e}") raise logger.debug("Loading Key Vault secrets into environment variables.") - print("loading keyvault vars") load_keyvault_vars(secret_client, force_keyvault=force_keyvault) From d92fbe84b15cfa8316b635e0e0dcac080f70e54b Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Mon, 5 Jan 2026 19:57:34 +0000 Subject: [PATCH 23/27] remove prints --- cfa/cloudops/_cloudclient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index 965806e..b0c2076 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -99,8 +99,7 @@ def __init__( **kwargs, ): logger.debug("Initializing CloudClient.") - if keyvault: - print(keyvault) + if keyvault is None and not os.path.exists(".env") and dotenv_path is None: try: keyvault = os.environ["AZURE_KEYVAULT_NAME"] From 6ca8b2b1934074252e994f34c713b312f63e0201 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 6 Jan 2026 16:16:54 +0000 Subject: [PATCH 24/27] save keyvault to env var --- cfa/cloudops/auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index 15b3e6f..a2f4530 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -1077,6 +1077,8 @@ def get_keyvault_vars( if keyvault_name is None: logger.debug("No Key Vault name provided; skipping Key Vault variable loading.") return None + else: + os.environ["AZURE_KEYVAULT_NAME"] = keyvault_name logger.debug("Getting SecretClient for Azure Key Vault.") try: secret_client = get_secret_client( From 7f0a13c437e95f54ff46b2fa51bf0be64fd48cc5 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 6 Jan 2026 20:30:17 +0000 Subject: [PATCH 25/27] change default dotenv path --- cfa/cloudops/_cloudclient.py | 2 +- cfa/cloudops/auth.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index b0c2076..94aaa7f 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -92,7 +92,7 @@ class CloudClient: def __init__( self, keyvault: str = None, - dotenv_path: str = None, + dotenv_path: str = ".env", use_sp: bool = False, use_federated: bool = False, force_keyvault: bool = False, diff --git a/cfa/cloudops/auth.py b/cfa/cloudops/auth.py index a2f4530..0b740c1 100644 --- a/cfa/cloudops/auth.py +++ b/cfa/cloudops/auth.py @@ -535,7 +535,7 @@ class EnvCredentialHandler(CredentialHandler): def __init__( self, - dotenv_path: str = None, + dotenv_path: str = ".env", keyvault: str = None, force_keyvault: bool = False, **kwargs, @@ -618,7 +618,7 @@ def __init__( azure_subscription_id: str = None, azure_client_id: str = None, azure_client_secret: str = None, - dotenv_path: str = None, + dotenv_path: str = ".env", keyvault: str = None, force_keyvault: bool = False, **kwargs, @@ -742,7 +742,7 @@ def __init__( class DefaultCredentialHandler(CredentialHandler): def __init__( self, - dotenv_path: str | None = None, + dotenv_path: str | None = ".env", keyvault: str = None, force_keyvault: bool = False, **kwargs, From 1d15c65316bf3ab21531a8be61356ac98707fbe0 Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 6 Jan 2026 20:51:40 +0000 Subject: [PATCH 26/27] add dotenv check --- cfa/cloudops/_cloudclient.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index 94aaa7f..f1309a5 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -92,15 +92,16 @@ class CloudClient: def __init__( self, keyvault: str = None, - dotenv_path: str = ".env", + dotenv_path: str = None, use_sp: bool = False, use_federated: bool = False, force_keyvault: bool = False, **kwargs, ): logger.debug("Initializing CloudClient.") - - if keyvault is None and not os.path.exists(".env") and dotenv_path is None: + if keyvault is None: + dotenv_path = dotenv_path or ".env" + if keyvault is None and dotenv_path is None: try: keyvault = os.environ["AZURE_KEYVAULT_NAME"] except KeyError: From b2963bfb9ee915307e12aec31fcb76f8bef8c95d Mon Sep 17 00:00:00 2001 From: Ryan Raasch Date: Tue, 6 Jan 2026 20:57:55 +0000 Subject: [PATCH 27/27] remove keyvault check --- cfa/cloudops/_cloudclient.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cfa/cloudops/_cloudclient.py b/cfa/cloudops/_cloudclient.py index f1309a5..57a0fc6 100644 --- a/cfa/cloudops/_cloudclient.py +++ b/cfa/cloudops/_cloudclient.py @@ -49,6 +49,7 @@ class CloudClient: provides convenient methods for common batch operations. Args: + keyvault (str, optional): Name of the Azure Key Vault to use for secrets. dotenv_path (str, optional): Path to .env file containing environment variables. If None, uses default .env file discovery. Default is None. use_sp (bool, optional): Whether to use Service Principal authentication. @@ -101,11 +102,6 @@ def __init__( logger.debug("Initializing CloudClient.") if keyvault is None: dotenv_path = dotenv_path or ".env" - if keyvault is None and dotenv_path is None: - try: - keyvault = os.environ["AZURE_KEYVAULT_NAME"] - except KeyError: - logger.error("Keyvault information not found.") if keyvault is None and force_keyvault: logger.error( "Keyvault information not found but force_keyvault set to True."