feat: support Nested Lists

core/src/main/java/io/substrait/expression/AbstractExpressionVisitor.java

@@ -186,6 +186,11 @@ public O visit(Expression.MultiOrList expr, C context) throws E {
     return visitFallback(expr, context);
   }
 
+  @Override
+  public O visit(Expression.NestedList expr, C context) throws E {
+    return visitFallback(expr, context);
+  }
+
   @Override
   public O visit(FieldReference expr, C context) throws E {
     return visitFallback(expr, context);

core/src/main/java/io/substrait/expression/Expression.java

@@ -32,6 +32,13 @@ default boolean nullable() {
     }
   }
 
+  interface Nested extends Expression {
+    @Value.Default
+    default boolean nullable() {
+      return false;
+    }
+  }
+
   <R, C extends VisitationContext, E extends Throwable> R accept(
       ExpressionVisitor<R, C, E> visitor, C context) throws E;
 
@@ -922,6 +929,40 @@ public <R, C extends VisitationContext, E extends Throwable> R accept(
     }
   }
 
+  /**
+   * A nested list expression with one or more elements.
+   *
+   * <p>Note: This class cannot be used to construct an empty list. To create an empty list, use
+   * {@link ExpressionCreator#emptyList(boolean, Type)} which returns an {@link EmptyListLiteral}.
+   */
+  @Value.Immutable
+  abstract class NestedList implements Nested {
+    public abstract List<Expression> values();
+
+    @Value.Check
+    protected void check() {
+      assert !values().isEmpty() : "To specify an empty list, use ExpressionCreator.emptyList()";
+
+      assert values().stream().map(Expression::getType).distinct().count() <= 1
+          : "All values in NestedList must have the same type";
+    }
+
+    @Override
+    public Type getType() {
+      return Type.withNullability(nullable()).list(values().get(0).getType());
+    }
+
+    @Override
+    public <R, C extends VisitationContext, E extends Throwable> R accept(
+        ExpressionVisitor<R, C, E> visitor, C context) throws E {
+      return visitor.visit(this, context);
+    }
+
+    public static ImmutableExpression.NestedList.Builder builder() {
+      return ImmutableExpression.NestedList.builder();
+    }
+  }
+
   @Value.Immutable
   abstract class MultiOrListRecord {
     public abstract List<Expression> values();

core/src/main/java/io/substrait/expression/ExpressionCreator.java

@@ -281,6 +281,17 @@ public static Expression.StructLiteral struct(boolean nullable, Expression.Liter
     return Expression.StructLiteral.builder().nullable(nullable).addFields(values).build();
   }
 
+  /**
+   * Creator a nested list expression with one or more elements.
+   *
+   * <p>Note: This class cannot be used to construct an empty list. To create an empty list, use
+   * {@link ExpressionCreator#emptyList(boolean, Type)} which returns an {@link
+   * Expression.EmptyListLiteral}.
+   */
+  public static Expression.NestedList nestedList(boolean nullable, List<Expression> values) {
+    return Expression.NestedList.builder().nullable(nullable).addAllValues(values).build();
+  }
+
   public static Expression.StructLiteral struct(
       boolean nullable, Iterable<? extends Expression.Literal> values) {
     return Expression.StructLiteral.builder().nullable(nullable).addAllFields(values).build();

core/src/main/java/io/substrait/expression/ExpressionVisitor.java

@@ -78,6 +78,8 @@ public interface ExpressionVisitor<R, C extends VisitationContext, E extends Thr
 
   R visit(Expression.MultiOrList expr, C context) throws E;
 
+  R visit(Expression.NestedList expr, C context) throws E;
+
   R visit(FieldReference expr, C context) throws E;
 
   R visit(Expression.SetPredicate expr, C context) throws E;

core/src/main/java/io/substrait/expression/proto/ExpressionProtoConverter.java

@@ -496,6 +496,22 @@ public Expression visit(
         .build();
   }
 
+  @Override
+  public Expression visit(
+      io.substrait.expression.Expression.NestedList expr, EmptyVisitationContext context)
+      throws RuntimeException {
+
+    List<Expression> values =
+        expr.values().stream().map(this::toProto).collect(Collectors.toList());
+
+    return Expression.newBuilder()
+        .setNested(
+            Expression.Nested.newBuilder()
+                .setList(Expression.Nested.List.newBuilder().addAllValues(values))
+                .setNullable(expr.nullable()))
+        .build();
+  }
+
   @Override
   public Expression visit(FieldReference expr, EmptyVisitationContext context) {
 

core/src/main/java/io/substrait/expression/proto/ProtoExpressionConverter.java

@@ -197,6 +197,8 @@ public Expression from(io.substrait.proto.Expression expr) {
                   multiOrList.getValueList().stream().map(this::from).collect(Collectors.toList()))
               .build();
         }
+      case NESTED:
+        return from(expr.getNested());
       case CAST:
         return ExpressionCreator.cast(
             protoTypeConverter.from(expr.getCast().getType()),
@@ -361,6 +363,17 @@ private WindowBound toWindowBound(io.substrait.proto.Expression.WindowFunction.B
     }
   }
 
+  public Expression.Nested from(io.substrait.proto.Expression.Nested nested) {
+    switch (nested.getNestedTypeCase()) {
+      case LIST:
+        List<Expression> list =
+            nested.getList().getValuesList().stream().map(this::from).collect(Collectors.toList());
+        return ExpressionCreator.nestedList(nested.getNullable(), list);
+      default:
+        throw new IllegalStateException("Unimplemented nested type: " + nested.getNestedTypeCase());
+    }
+  }
+
   public Expression.Literal from(io.substrait.proto.Expression.Literal literal) {
     switch (literal.getLiteralTypeCase()) {
       case BOOLEAN:

core/src/main/java/io/substrait/relation/ExpressionCopyOnWriteVisitor.java

@@ -348,6 +348,16 @@ public Optional<Expression> visit(
             .build());
   }
 
+  @Override
+  public Optional<Expression> visit(Expression.NestedList expr, EmptyVisitationContext context)
+      throws E {
+    Optional<List<Expression>> expressions = visitExprList(expr.values(), context);
+
+    return expressions.map(
+        expressionList ->
+            Expression.NestedList.builder().from(expr).values(expressionList).build());
+  }
+
   protected Optional<Expression.MultiOrListRecord> visitMultiOrListRecord(
       Expression.MultiOrListRecord multiOrListRecord, EmptyVisitationContext context) throws E {
     return visitExprList(multiOrListRecord.values(), context)

core/src/test/java/io/substrait/type/proto/NestedListExpressionTest.java

@@ -0,0 +1,94 @@
+package io.substrait.type.proto;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import io.substrait.TestBase;
+import io.substrait.expression.Expression;
+import io.substrait.expression.ImmutableExpression;
+import org.junit.jupiter.api.Test;
+
+class NestedListExpressionTest extends TestBase {
+  io.substrait.expression.Expression literalExpression =
+      Expression.BoolLiteral.builder().value(true).build();
+  Expression.ScalarFunctionInvocation nonLiteralExpression = b.add(b.i32(7), b.i32(42));
+
+  @Test
+  void DifferentTypedLiteralsNestedListTest() {
+    ImmutableExpression.NestedList.Builder builder =
+        Expression.NestedList.builder().addValues(literalExpression).addValues(b.i32(12));
+    assertThrows(AssertionError.class, builder::build);
+  }
+
+  @Test
+  void SameTypedLiteralsNestedListTest() {
+    ImmutableExpression.NestedList.Builder builder =
+        Expression.NestedList.builder().addValues(nonLiteralExpression).addValues(b.i32(12));
+    assertDoesNotThrow(builder::build);
+
+    io.substrait.relation.Project project =
+        io.substrait.relation.Project.builder()
+            .addExpressions(builder.build())
+            .input(b.emptyScan())
+            .build();
+    verifyRoundTrip(project);
+  }
+
+  @Test
+  void EmptyNestedListTest() {
+    ImmutableExpression.NestedList.Builder builder = Expression.NestedList.builder();
+    assertThrows(AssertionError.class, builder::build);
+  }
+
+  @Test
+  void literalNestedListTest() {
+    Expression.NestedList literalNestedList =
+        Expression.NestedList.builder()
+            .addValues(literalExpression)
+            .addValues(literalExpression)
+            .build();
+
+    io.substrait.relation.Project project =
+        io.substrait.relation.Project.builder()
+            .addExpressions(literalNestedList)
+            .input(b.emptyScan())
+            .build();
+
+    verifyRoundTrip(project);
+  }
+
+  @Test
+  void literalNullableNestedListTest() {
+    Expression.NestedList literalNestedList =
+        Expression.NestedList.builder()
+            .addValues(literalExpression)
+            .addValues(literalExpression)
+            .nullable(true)
+            .build();
+
+    io.substrait.relation.Project project =
+        io.substrait.relation.Project.builder()
+            .addExpressions(literalNestedList)
+            .input(b.emptyScan())
+            .build();
+
+    verifyRoundTrip(project);
+  }
+
+  @Test
+  void nonLiteralNestedListTest() {
+    Expression.NestedList nonLiteralNestedList =
+        Expression.NestedList.builder()
+            .addValues(nonLiteralExpression)
+            .addValues(nonLiteralExpression)
+            .build();
+
+    io.substrait.relation.Project project =
+        io.substrait.relation.Project.builder()
+            .addExpressions(nonLiteralNestedList)
+            .input(b.emptyScan())
+            .build();
+
+    verifyRoundTrip(project);
+  }
+}

examples/substrait-spark/src/main/java/io/substrait/examples/util/ExpressionStringify.java

@@ -1,5 +1,6 @@
 package io.substrait.examples.util;
 
+import io.substrait.expression.Expression;
 import io.substrait.expression.Expression.BinaryLiteral;
 import io.substrait.expression.Expression.BoolLiteral;
 import io.substrait.expression.Expression.Cast;
@@ -256,6 +257,12 @@ public String visit(MultiOrList expr, EmptyVisitationContext context) throws Run
     return sb.toString();
   }
 
+  @Override
+  public String visit(Expression.NestedList expr, EmptyVisitationContext context)
+      throws RuntimeException {
+    return "<NestedList>";
+  }
+
   @Override
   public String visit(FieldReference expr, EmptyVisitationContext context) throws RuntimeException {
     StringBuilder sb = new StringBuilder("FieldRef#");

isthmus/src/main/java/io/substrait/isthmus/NestedListConstructor.java

@@ -0,0 +1,34 @@
+package io.substrait.isthmus;
+
+import static java.util.Objects.requireNonNull;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.fun.SqlMultisetValueConstructor;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.validate.SqlValidatorUtil;
+
+/**
+ * Substrait-specific constructor to map back to the Expression NestedList type in Substrait. This
+ * constructor creates a special type of SqlKind.ARRAY_VALUE_CONSTRUCTOR for lists that store
+ * non-literal expressions.
+ */
+public class NestedListConstructor extends SqlMultisetValueConstructor {
+
+  public NestedListConstructor() {
+    super("NESTEDLIST", SqlKind.ARRAY_VALUE_CONSTRUCTOR);
+  }
+
+  @Override
+  public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+    RelDataType type =
+        getComponentType(opBinding.getTypeFactory(), opBinding.collectOperandTypes());
+    requireNonNull(type, "inferred array element type");
+
+    // explicit cast elements to component type if they are not same
+    SqlValidatorUtil.adjustTypeForArrayConstructor(type, opBinding);
+
+    return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), type, false);
+  }
+}

isthmus/src/main/java/io/substrait/isthmus/SubstraitRelVisitor.java

@@ -179,6 +179,10 @@ public Rel visit(org.apache.calcite.rel.core.Project project) {
             .map(this::toExpression)
             .collect(java.util.stream.Collectors.toList());
 
+    //    if there is no input fields, don’t put a remapping on it
+    if (project.getInput().getRowType().getFieldCount() == 0) {
+      return Project.builder().expressions(expressions).input(apply(project.getInput())).build();
+    }
     // todo: eliminate excessive projects. This should be done by converting rexinputrefs to remaps.
     return Project.builder()
         .remap(

isthmus/src/main/java/io/substrait/isthmus/calcite/SubstraitOperatorTable.java

@@ -1,6 +1,7 @@
 package io.substrait.isthmus.calcite;
 
 import io.substrait.isthmus.AggregateFunctions;
+import io.substrait.isthmus.NestedListConstructor;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Set;
@@ -36,6 +37,8 @@ public class SubstraitOperatorTable implements SqlOperatorTable {
               AggregateFunctions.SUM,
               AggregateFunctions.SUM0));
 
+  public static NestedListConstructor NESTED_LIST_CONSTRUCTOR = new NestedListConstructor();
+
   // SQL Kinds for which Substrait specific operators are provided
   private static final Set<SqlKind> OVERRIDE_KINDS =
       EnumSet.copyOf(

isthmus/src/main/java/io/substrait/isthmus/expression/CallConverters.java

@@ -141,7 +141,8 @@ public static List<CallConverter> defaults(TypeConverter typeConverter) {
         CallConverters.CASE,
         CallConverters.CAST.apply(typeConverter),
         CallConverters.REINTERPRET.apply(typeConverter),
-        new LiteralConstructorConverter(typeConverter));
+        new LiteralConstructorConverter(typeConverter),
+        new NestedExpressionConverter());
   }
 
   public interface SimpleCallConverter extends CallConverter {

isthmus/src/main/java/io/substrait/isthmus/expression/ExpressionRexConverter.java

@@ -22,6 +22,7 @@
 import io.substrait.isthmus.SubstraitRelNodeConverter;
 import io.substrait.isthmus.SubstraitRelNodeConverter.Context;
 import io.substrait.isthmus.TypeConverter;
+import io.substrait.isthmus.calcite.SubstraitOperatorTable;
 import io.substrait.type.StringTypeVisitor;
 import io.substrait.type.Type;
 import io.substrait.util.DecimalUtil;
@@ -320,6 +321,13 @@ public RexNode visit(Expression.ListLiteral expr, Context context) throws Runtim
     return rexBuilder.makeCall(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, args);
   }
 
+  @Override
+  public RexNode visit(Expression.NestedList expr, Context context) {
+    List<RexNode> args =
+        expr.values().stream().map(e -> e.accept(this, context)).collect(Collectors.toList());
+    return rexBuilder.makeCall(SubstraitOperatorTable.NESTED_LIST_CONSTRUCTOR, args);
+  }
+
   @Override
   public RexNode visit(Expression.EmptyListLiteral expr, Context context) throws RuntimeException {
     RelDataType calciteType = typeConverter.toCalcite(typeFactory, expr.getType());