Mvexpand feature

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -80,6 +80,7 @@
 import org.opensearch.sql.ast.tree.Lookup;
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Multisearch;
+import org.opensearch.sql.ast.tree.MvExpand;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
@@ -713,6 +714,11 @@ public LogicalPlan visitExpand(Expand expand, AnalysisContext context) {
     throw getOnlyForCalciteException("Expand");
   }
 
+  @Override
+  public LogicalPlan visitMvExpand(MvExpand node, AnalysisContext context) {
+    throw getOnlyForCalciteException("MvExpand");
+  }
+
   /** Build {@link LogicalTrendline} for Trendline command. */
   @Override
   public LogicalPlan visitTrendline(Trendline node, AnalysisContext context) {

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -68,6 +68,7 @@
 import org.opensearch.sql.ast.tree.Lookup;
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Multisearch;
+import org.opensearch.sql.ast.tree.MvExpand;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
@@ -461,4 +462,8 @@ public T visitAddTotals(AddTotals node, C context) {
   public T visitAddColTotals(AddColTotals node, C context) {
     return visitChildren(node, context);
   }
+
+  public T visitMvExpand(MvExpand node, C context) {
+    return visitChildren(node, context);
+  }
 }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -62,6 +62,7 @@
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.MinSpanBin;
+import org.opensearch.sql.ast.tree.MvExpand;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
@@ -136,6 +137,10 @@ public Expand expand(UnresolvedPlan input, Field field, String alias) {
     return new Expand(field, alias).attach(input);
   }
 
+  public static UnresolvedPlan mvexpand(UnresolvedPlan input, Field field, Integer limit) {
+    return new MvExpand(field, limit).attach(input);
+  }
+
   public static UnresolvedPlan projectWithArg(
       UnresolvedPlan input, List<Argument> argList, UnresolvedExpression... projectList) {
     return new Project(Arrays.asList(projectList), argList).attach(input);

core/src/main/java/org/opensearch/sql/ast/tree/MvExpand.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import javax.annotation.Nullable;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+/** AST node representing the {@code mvexpand} PPL command: {@code mvexpand <field> [limit=N]}. */
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class MvExpand extends UnresolvedPlan {
+
+  private UnresolvedPlan child;
+  @Getter private final Field field;
+  @Getter @Nullable private final Integer limit;
+
+  public MvExpand(Field field, @Nullable Integer limit) {
+    this.field = field;
+    this.limit = limit;
+  }
+
+  @Override
+  public MvExpand attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitMvExpand(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -51,6 +51,7 @@
 import org.apache.calcite.rel.core.JoinRelType;
 import org.apache.calcite.rel.logical.LogicalValues;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeFamily;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexCall;
@@ -122,6 +123,7 @@
 import org.opensearch.sql.ast.tree.Lookup.OutputStrategy;
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Multisearch;
+import org.opensearch.sql.ast.tree.MvExpand;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
@@ -846,7 +848,11 @@ public RelNode visitPatterns(Patterns node, CalcitePlanContext context) {
                 .toList();
         context.relBuilder.aggregate(context.relBuilder.groupKey(groupByList), aggCall);
         buildExpandRelNode(
-            context.relBuilder.field(node.getAlias()), node.getAlias(), node.getAlias(), context);
+            context.relBuilder.field(node.getAlias()),
+            node.getAlias(),
+            node.getAlias(),
+            null,
+            context);
         flattenParsedPattern(
             node.getAlias(),
             context.relBuilder.field(node.getAlias()),
@@ -3111,11 +3117,82 @@ public RelNode visitExpand(Expand expand, CalcitePlanContext context) {
     RexInputRef arrayFieldRex = (RexInputRef) rexVisitor.analyze(arrayField, context);
     String alias = expand.getAlias();
 
-    buildExpandRelNode(arrayFieldRex, arrayField.getField().toString(), alias, context);
+    buildExpandRelNode(arrayFieldRex, arrayField.getField().toString(), alias, null, context);
 
     return context.relBuilder.peek();
   }
 
+  /**
+   * MVExpand command visitor.
+   *
+   * <p>For Calcite remote planning, mvexpand shares the same expansion mechanics as {@link Expand}:
+   * it unnests the target multivalue field and joins back to the original relation. The additional
+   * mvexpand semantics (such as an optional per-document limit) are surfaced via the MVExpand AST
+   * node but reuse the same underlying RelBuilder pipeline as expand at this layer.
+   *
+   * @param mvExpand MVExpand command to be visited
+   * @param context CalcitePlanContext containing the RelBuilder and other context
+   * @return RelNode representing records with the expanded multi-value field
+   */
+  /**
+   * MVExpand command visitor.
+   *
+   * <p>For Calcite remote planning, mvexpand reuses the same expansion mechanics as {@link Expand}:
+   * it unnests the target multivalue field and joins back to the original relation.
+   * mvexpand-specific semantics (such as an optional per-document limit) are carried by the {@link
+   * MvExpand} AST node and applied via the limit parameter passed into the shared expansion
+   * builder.
+   *
+   * <p>Missing-field behavior: if the target field does not exist in the input schema, mvexpand
+   * produces no rows while keeping the output schema stable.
+   *
+   * @param mvExpand MVExpand command to be visited
+   * @param context CalcitePlanContext containing the RelBuilder and other context
+   * @return RelNode representing records with the expanded multi-value field
+   */
+  @Override
+  public RelNode visitMvExpand(MvExpand mvExpand, CalcitePlanContext context) {
+    visitChildren(mvExpand, context);
+
+    final RelBuilder relBuilder = context.relBuilder;
+    final Field field = mvExpand.getField();
+    final String fieldName = field.getField().toString();
+
+    // Missing-field: produce no rows (but keep schema stable).
+    final RelDataType inputType = relBuilder.peek().getRowType();
+    final RelDataTypeField inputField =
+        inputType.getField(fieldName, /*caseSensitive*/ false, /*elideRecord*/ false);
+    if (inputField == null) {
+      return buildEmptyResultWithStableSchema(relBuilder, fieldName);
+    }
+
+    // Resolve field ref using rexVisitor for consistent semantics (same as expand).
+    final RexInputRef arrayFieldRex = (RexInputRef) rexVisitor.analyze(field, context);
+
+    // Enforce ARRAY type before UNNEST so we return SemanticCheckException.
+    final SqlTypeName actual = arrayFieldRex.getType().getSqlTypeName();
+    if (actual != SqlTypeName.ARRAY) {
+      throw new SemanticCheckException(
+          String.format(
+              "Cannot expand field '%s': expected ARRAY type but found %s", fieldName, actual));
+    }
+
+    buildExpandRelNode(arrayFieldRex, fieldName, fieldName, mvExpand.getLimit(), context);
+    return relBuilder.peek();
+  }
+
+  private static RelNode buildEmptyResultWithStableSchema(RelBuilder relBuilder, String fieldName) {
+    final RelDataTypeFactory typeFactory = relBuilder.getTypeFactory();
+    final RelDataType arrayAny =
+        typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.ANY), -1);
+
+    relBuilder.projectPlus(
+        List.of(relBuilder.alias(relBuilder.getRexBuilder().makeNullLiteral(arrayAny), fieldName)));
+
+    relBuilder.filter(relBuilder.literal(false));
+    return relBuilder.peek();
+  }
+
   @Override
   public RelNode visitValues(Values values, CalcitePlanContext context) {
     if (values.getValues() == null || values.getValues().isEmpty()) {
@@ -3360,7 +3437,11 @@ private void flattenParsedPattern(
   }
 
   private void buildExpandRelNode(
-      RexInputRef arrayFieldRex, String arrayFieldName, String alias, CalcitePlanContext context) {
+      RexInputRef arrayFieldRex,
+      String arrayFieldName,
+      String alias,
+      @Nullable Integer perDocLimit,
+      CalcitePlanContext context) {
     // 3. Capture the outer row in a CorrelationId
     Holder<RexCorrelVariable> correlVariable = Holder.empty();
     context.relBuilder.variable(correlVariable::set);
@@ -3375,14 +3456,17 @@ private void buildExpandRelNode(
     RelNode leftNode = context.relBuilder.build();
 
     // 5. Build join right node and expand the array field using uncollect
-    RelNode rightNode =
-        context
-            .relBuilder
-            // fake input, see convertUnnest and convertExpression in Calcite SqlToRelConverter
-            .push(LogicalValues.createOneRow(context.relBuilder.getCluster()))
-            .project(List.of(correlArrayFieldAccess), List.of(arrayFieldName))
-            .uncollect(List.of(), false)
-            .build();
+    context
+        .relBuilder
+        // fake input, see convertUnnest and convertExpression in Calcite SqlToRelConverter
+        .push(LogicalValues.createOneRow(context.relBuilder.getCluster()))
+        .project(List.of(correlArrayFieldAccess), List.of(arrayFieldName))
+        .uncollect(List.of(), false);
+
+    if (perDocLimit != null) {
+      context.relBuilder.limit(0, perDocLimit);
+    }
+    RelNode rightNode = context.relBuilder.build();
 
     // 6. Perform a nested-loop join (correlate) between the original table and the expanded
     // array field.

core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

@@ -1108,6 +1108,14 @@ void populate() {
                   OperandTypes.family(SqlTypeFamily.ARRAY, SqlTypeFamily.INTEGER)
                       .or(OperandTypes.family(SqlTypeFamily.MAP, SqlTypeFamily.ANY)),
               false));
+      // Allow using INTERNAL_ITEM when the element type is unknown/undefined at planning time.
+      // Some datasets (or Calcite's type inference) may give the element an UNDEFINED type.
+      // Accept a "ignore" first-argument family so INTERNAL_ITEM(elem, 'key') can still be planned
+      // and resolved at runtime (fallback semantics handled at execution side). - Used in MVEXPAND
+      registerOperator(
+          INTERNAL_ITEM,
+          SqlStdOperatorTable.ITEM,
+          PPLTypeChecker.family(SqlTypeFamily.IGNORE, SqlTypeFamily.CHARACTER));
       registerOperator(
           XOR,
           SqlStdOperatorTable.NOT_EQUALS,

docs/category.json

@@ -29,6 +29,7 @@
     "user/ppl/cmd/regex.md",
     "user/ppl/cmd/rename.md",
     "user/ppl/cmd/multisearch.md",
+    "user/ppl/cmd/mvexpand.md",
     "user/ppl/cmd/replace.md",
     "user/ppl/cmd/rex.md",
     "user/ppl/cmd/search.md",
@@ -79,4 +80,4 @@
   "bash_settings": [
     "user/ppl/admin/settings.md"
   ]
-}
+}

docs/user/dql/metadata.rst

@@ -35,7 +35,7 @@ Example 1: Show All Indices Information
 SQL query::
 
     os> SHOW TABLES LIKE '%'
-    fetched rows / total rows = 23/23
+    fetched rows / total rows = 24/24
     +----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------+
     | TABLE_CAT      | TABLE_SCHEM | TABLE_NAME        | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION |
     |----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------|
@@ -48,6 +48,7 @@ SQL query::
     | docTestCluster | null        | events_many_hosts | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | events_null       | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | json_test         | BASE TABLE | null    | null     | null       | null      | null                      | null           |
+    | docTestCluster | null        | mvexpand_logs     | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | nested            | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | nyc_taxi          | BASE TABLE | null    | null     | null       | null      | null                      | null           |
     | docTestCluster | null        | occupation        | BASE TABLE | null    | null     | null       | null      | null                      | null           |
@@ -120,3 +121,4 @@ SQL query::
     | docTestCluster | null        | accounts   | firstname   | null      | text      | null        | null          | null           | 10             | 2        | null    | null       | null          | null             | null              | 1                |             | null          | null         | null        | null             | NO               |                    |
     | docTestCluster | null        | accounts   | lastname    | null      | text      | null        | null          | null           | 10             | 2        | null    | null       | null          | null             | null              | 10               |             | null          | null         | null        | null             | NO               |                    |
     +----------------+-------------+------------+-------------+-----------+-----------+-------------+---------------+----------------+----------------+----------+---------+------------+---------------+------------------+-------------------+------------------+-------------+---------------+--------------+-------------+------------------+------------------+--------------------+
+