[BACKLOG-48717] - list & test project support

- add "allLevels" parameter
- add "level" to response database objects in list call
- support Variables in test

Author: abryant-hv

api/src/main/resources/connection-api.yaml

@@ -289,6 +289,12 @@ paths:
           description: Optional project directory. If provided, the operation will be performed in the context of the project.
           schema:
             type: string
+        - name: allLevels
+          in: query
+          required: false
+          description: Optional flag to include connections from all supported levels for the request context.
+          schema:
+            type: boolean
       responses:
         '200':
           description: Successfully retrieved connections
@@ -568,6 +574,10 @@ components:
           description: Partitioning information
           items:
             type: object
+        level:
+          type: string
+          description: Connection level for list results, present when allLevels=true
+          example: "Repository"
     IDatabaseConnectionList:
       type: object
       description: List of database connections (maps to org.pentaho.ui.database.event.IDatabaseConnectionList)

core/src/main/java/org/pentaho/platform/dataaccess/datasource/utils/DatabaseConnectionUtils.java

@@ -0,0 +1,132 @@
+package org.pentaho.platform.dataaccess.datasource.utils;
+
+import org.pentaho.database.model.IDatabaseConnection;
+import org.pentaho.database.model.PartitionDatabaseMeta;
+import org.pentaho.di.core.variables.VariableSpace;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Utility class for applying variable substitution to database connection fields.
+ * Applies environmentSubstitute to connection fields that may contain variable references
+ * (in the form ${VARIABLE_NAME}).
+ */
+public class DatabaseConnectionUtils {
+
+  /**
+   * Apply environment variable substitution to an IDatabaseConnection using the provided VariableSpace.
+   * Substitutes variables in the following fields:
+   * - name (connection identifier, may be used in pooling keys)
+   * - hostname
+   * - databaseName
+   * - databasePort
+   * - username
+   * - password
+   * - connectSql
+   * - sqlServerInstance
+   * - dataTablespace (Oracle and other database-specific tablespaces)
+   * - indexTablespace (Oracle and other database-specific tablespaces)
+   * - connectionPoolingProperties (all values)
+   * - partitioning information (if present)
+   *
+   * @param connection The database connection to update (modified in-place)
+   * @param variableSpace The variable space to use for substitution
+   */
+  public static void applyEnvironmentSubstitution( final IDatabaseConnection connection,
+      final VariableSpace variableSpace ) {
+    if ( connection == null || variableSpace == null ) {
+      return;
+    }
+
+    // Substitute connection name (used for pooling and identification)
+    if ( connection.getName() != null ) {
+      connection.setName( variableSpace.environmentSubstitute( connection.getName() ) );
+    }
+
+    // Substitute basic connection fields
+    if ( connection.getHostname() != null ) {
+      connection.setHostname( variableSpace.environmentSubstitute( connection.getHostname() ) );
+    }
+
+    if ( connection.getDatabaseName() != null ) {
+      connection.setDatabaseName( variableSpace.environmentSubstitute( connection.getDatabaseName() ) );
+    }
+
+    if ( connection.getDatabasePort() != null ) {
+      connection.setDatabasePort( variableSpace.environmentSubstitute( connection.getDatabasePort() ) );
+    }
+
+    if ( connection.getUsername() != null ) {
+      connection.setUsername( variableSpace.environmentSubstitute( connection.getUsername() ) );
+    }
+
+    if ( connection.getPassword() != null ) {
+      connection.setPassword( variableSpace.environmentSubstitute( connection.getPassword() ) );
+    }
+
+    // Substitute connect SQL
+    if ( connection.getConnectSql() != null ) {
+      connection.setConnectSql( variableSpace.environmentSubstitute( connection.getConnectSql() ) );
+    }
+
+    // Substitute SQL Server instance
+    if ( connection.getSQLServerInstance() != null ) {
+      connection.setSQLServerInstance( variableSpace.environmentSubstitute( connection.getSQLServerInstance() ) );
+    }
+
+    // Substitute tablespace fields (Oracle and similar databases)
+    if ( connection.getDataTablespace() != null ) {
+      connection.setDataTablespace( variableSpace.environmentSubstitute( connection.getDataTablespace() ) );
+    }
+
+    if ( connection.getIndexTablespace() != null ) {
+      connection.setIndexTablespace( variableSpace.environmentSubstitute( connection.getIndexTablespace() ) );
+    }
+
+    // Substitute connection pooling properties
+    if ( connection.getConnectionPoolingProperties() != null
+        && !connection.getConnectionPoolingProperties().isEmpty() ) {
+      Map<String, String> substitutedPoolingProps = new HashMap<>();
+      for ( Map.Entry<String, String> entry : connection.getConnectionPoolingProperties().entrySet() ) {
+        String substitutedValue = entry.getValue() != null ? variableSpace.environmentSubstitute( entry.getValue() )
+            : null;
+        substitutedPoolingProps.put( entry.getKey(), substitutedValue );
+      }
+      connection.setConnectionPoolingProperties( substitutedPoolingProps );
+    }
+
+    // Substitute partitioning information if present
+    if ( connection.isPartitioned() && connection.getPartitioningInformation() != null ) {
+      for ( PartitionDatabaseMeta partition : connection.getPartitioningInformation() ) {
+        applyEnvironmentSubstitutionToPartition( partition, variableSpace );
+      }
+    }
+  }
+
+  /**
+   * Apply environment variable substitution to a PartitionDatabaseMeta.
+   * Substitutes variables in hostname, port, and database name fields.
+   *
+   * @param partition The partition metadata to update (modified in-place)
+   * @param variableSpace The variable space to use for substitution
+   */
+  private static void applyEnvironmentSubstitutionToPartition( final PartitionDatabaseMeta partition,
+      final VariableSpace variableSpace ) {
+    if ( partition == null || variableSpace == null ) {
+      return;
+    }
+
+    if ( partition.getHostname() != null ) {
+      partition.setHostname( variableSpace.environmentSubstitute( partition.getHostname() ) );
+    }
+
+    if ( partition.getPort() != null ) {
+      partition.setPort( variableSpace.environmentSubstitute( partition.getPort() ) );
+    }
+
+    if ( partition.getDatabaseName() != null ) {
+      partition.setDatabaseName( variableSpace.environmentSubstitute( partition.getDatabaseName() ) );
+    }
+  }
+}

core/src/main/java/org/pentaho/platform/dataaccess/datasource/wizard/service/impl/ConnectionService.java

@@ -22,8 +22,11 @@
 import org.pentaho.database.service.DatabaseDialectService;
 import org.pentaho.database.util.DatabaseUtil;
 import org.pentaho.di.core.encryption.Encr;
+import org.pentaho.di.core.variables.VariableSpace;
+import org.pentaho.di.core.variables.Variables;
 import org.pentaho.platform.api.engine.IAuthorizationPolicy;
 import org.pentaho.platform.api.engine.PentahoAccessControlException;
+import org.pentaho.platform.dataaccess.datasource.utils.DatabaseConnectionUtils;
 import org.pentaho.platform.dataaccess.datasource.wizard.service.ConnectionServiceException;
 import org.pentaho.platform.dataaccess.datasource.wizard.service.api.ConnectionsApi;
 import org.pentaho.platform.dataaccess.datasource.wizard.service.impl.utils.UtilHtmlSanitizer;
@@ -61,6 +64,7 @@ public class ConnectionService implements ConnectionsApi {
   private static final Log logger = LogFactory.getLog( ConnectionService.class );
   protected static final String MEDIA_TYPE_JSON = "application/json";
   protected static final String MEDIA_TYPE_TEXT_PLAIN = "text/plain";
+  protected static final String CONNECTION_LEVEL_REPOSITORY = "Repository";
 
   private ConnectionServiceImpl connectionService;
   private DatabaseDialectService dialectService;
@@ -283,7 +287,7 @@ public IDatabaseConnectionPoolParameterList getPoolingParameters() {
   @Override
   public String testConnection( IDatabaseConnection databaseConnection, String projectDir ) {
     try {
-      applySavedPassword( databaseConnection, projectDir );
+      applySavedPassword( databaseConnection, projectDir, true );
       boolean success = connectionService.testConnection( databaseConnection );
       if ( success ) {
         return Messages.getString( "ConnectionServiceImpl.INFO_0001_CONNECTION_SUCCEED", databaseConnection
@@ -319,7 +323,7 @@ public String testConnection( IDatabaseConnection databaseConnection ) {
   public void updateConnection( IDatabaseConnection databaseConnection, String projectDir ) {
     sanitizer.sanitizeConnectionParameters( databaseConnection );
     try {
-      applySavedPassword( databaseConnection );
+      applySavedPassword( databaseConnection, false );
       connectionService.updateConnection( databaseConnection );
       // explicitly return a 200 instead of 204 No Content
       throw new WebApplicationException( Response.ok().build() );
@@ -358,12 +362,22 @@ public void updateConnection( IDatabaseConnection databaseConnection ) {
   /**
    * If password is empty, that means connection sent from UI and user didn't change password. Since we cleaned password
    * during sending to UI, we need to use stored password.
+   * 
+   * @param conn the connection to update 
+   * @param projectDir Optional project directory (used by subclasses)
+   * @param resolveVariables whether to resolve variables in the connection
    */
-  protected void applySavedPassword( IDatabaseConnection conn, String projectDir ) throws ConnectionServiceException {
-    applySavedPassword( conn );
+  protected void applySavedPassword( IDatabaseConnection conn, String projectDir, boolean resolveVariables ) throws ConnectionServiceException {
+    applySavedPassword( conn, resolveVariables );
   }
 
-  private void applySavedPassword( IDatabaseConnection conn ) throws ConnectionServiceException {
+  private void applySavedPassword( IDatabaseConnection conn, boolean resolveVariables ) throws ConnectionServiceException {
+
+    if ( resolveVariables ) {
+      VariableSpace variables = Variables.getADefaultVariableSpace();
+      DatabaseConnectionUtils.applyEnvironmentSubstitution( conn, variables );
+    }
+    
     if ( StringUtils.isBlank( conn.getPassword() ) ) {
       IDatabaseConnection savedConn;
       if ( conn.getId() != null ) {
@@ -519,19 +533,30 @@ private void validateAccess() throws PentahoAccessControlException {
    *
    */
   @Override
-  public IDatabaseConnectionList getConnections( String projectDir ) {
+  public IDatabaseConnectionList getConnections( String projectDir, Boolean allLevels ) {
     try {
       IDatabaseConnectionList databaseConnections = new DefaultDatabaseConnectionList();
       List<IDatabaseConnection> conns = connectionService.getConnections( true );
+      for ( IDatabaseConnection conn : conns ) {
+        setConnectionLevel( conn, CONNECTION_LEVEL_REPOSITORY );
+      }
       databaseConnections.setDatabaseConnections( conns );
       return databaseConnections;
     } catch ( ConnectionServiceException ex ) {
       throw new WebApplicationException( ex.getMessage(), Response.Status.INTERNAL_SERVER_ERROR );
     }
   }
 
+  public IDatabaseConnectionList getConnections( String projectDir ) {
+    return getConnections( projectDir, Boolean.FALSE );
+  }
+
   public IDatabaseConnectionList getConnections() {
-    return getConnections( null );
+    return getConnections( null, Boolean.FALSE );
+  }
+
+  protected void setConnectionLevel( IDatabaseConnection connection, String level ) {
+    connection.setLevel( level );
   }
 
   /**