Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,12 @@ private List<Node> processQueryDefinition(ObjectTypeDefinition definition, Docum
List<Node> queries = new ArrayList<>();
var defs = definition.getFieldDefinitions();
for (FieldDefinition def : defs) {
final var tableFn = getTableFunctionFromPath(tableFunctions, def.getName()).get();
// Namespace fields (e.g. `backend`) have no table function at the root path; skip them.
final var tableFnOpt = getTableFunctionFromPath(tableFunctions, def.getName());
if (tableFnOpt.isEmpty()) {
continue;
}
final var tableFn = tableFnOpt.get();
if (tableFn.getVisibility().isTest()) {
var operation =
processOperation(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright © 2021 DataSQRL (contact@datasqrl.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.datasqrl.planner;

import com.datasqrl.canonicalizer.Name;
import com.datasqrl.server.ResolvedMetadata;
import java.util.List;
import java.util.Optional;
import org.apache.calcite.rel.type.RelDataType;

/**
* A {@code CREATE NAMESPACE} declaration: a name plus the parameters shared by every function in
* the namespace. A parameter with metadata is a hidden JWT claim; a parameter without metadata is
* an external argument exposed on the namespace field (e.g. {@code admin(asTenantId: String!)}).
*/
public record NamespaceDefinition(Name name, List<Param> params) {

public record Param(String name, RelDataType type, Optional<ResolvedMetadata> metadata) {
public boolean isExternal() {
return metadata.isEmpty();
}
}

public Optional<Param> getParam(String paramName) {
return params.stream().filter(p -> p.name().equalsIgnoreCase(paramName)).findFirst();
}

public List<Param> externalParams() {
return params.stream().filter(Param::isExternal).toList();
}
}
182 changes: 160 additions & 22 deletions sqrl-planner/src/main/java/com/datasqrl/planner/SqlScriptPlanner.java
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import com.datasqrl.planner.parser.SQLStatement;
import com.datasqrl.planner.parser.SqlScriptStatementSplitter;
import com.datasqrl.planner.parser.SqrlAddColumnStatement;
import com.datasqrl.planner.parser.SqrlCreateNamespaceStatement;
import com.datasqrl.planner.parser.SqrlCreateTableStatement;
import com.datasqrl.planner.parser.SqrlDefinition;
import com.datasqrl.planner.parser.SqrlExportStatement;
Expand All @@ -95,6 +96,7 @@
import com.datasqrl.planner.parser.StatementParserException;
import com.datasqrl.planner.tables.AccessVisibility;
import com.datasqrl.planner.tables.FlinkTableBuilder;
import com.datasqrl.planner.tables.SqrlFunctionParameter;
import com.datasqrl.planner.tables.SqrlTableFunction;
import com.datasqrl.planner.util.SqlScriptWriter;
import com.datasqrl.planner.util.SqlTableNameExtractor;
Expand All @@ -115,6 +117,7 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.schema.FunctionParameter;
Expand Down Expand Up @@ -144,13 +147,17 @@
*/
@Component
@Lazy
@Slf4j
public class SqlScriptPlanner {

private static final String EXPORT_SUFFIX = "_ex";
private static final String ACCESS_FUNCTION_SUFFIX = "__access";

private final AtomicInteger exportTableCounter = new AtomicInteger(0);

/** Declared namespaces (grouping + shared parameters), keyed by namespace name. */
private final Map<Name, NamespaceDefinition> namespaces = new HashMap<>();

private final ErrorCollector errorCollector;

/** Used to assemble the full script with imports as a string */
Expand Down Expand Up @@ -279,7 +286,8 @@ public void planMain(
throw lineErrors.handle(e);
}

if (!(sqlStatement instanceof SqrlImportStatement)) {
if (!(sqlStatement instanceof SqrlImportStatement)
&& !(sqlStatement instanceof SqrlCreateNamespaceStatement)) {
completeScript.append(sourceStmt.source());
}

Expand Down Expand Up @@ -339,6 +347,8 @@ private void planStatement(
scriptContext.mainModuleLoader().getSchemaLoader(),
hintsAndDocs)
.ifPresent(tableAnalysis -> addSourceToDag(tableAnalysis, hintsAndDocs, sqrlEnv));
} else if (stmt instanceof SqrlCreateNamespaceStatement nsStmt) {
addNamespace(nsStmt, sqrlEnv, errors);
} else if (stmt instanceof SqrlDefinition sqrlDef) {
var access = sqrlDef.getAccess();
var tablePath = sqrlDef.getPath();
Expand Down Expand Up @@ -411,30 +421,45 @@ private void planStatement(
});
}
TableAnalysis parentTbl = null;
var namespaced = false;
final Name namespaceName =
tblFnStmt.isRelationship() ? tblFnStmt.getPath().getFirst() : null;
NamespaceDefinition namespaceDef = null;
if (tblFnStmt.isRelationship()) {
/* To resolve the arguments and get their type, we first need to look up the parent table
/* A size-2 path is either a relationship on an existing table or a namespaced root
function grouped under a namespace (e.g. `backend.dormantDeployments`). If the head
resolves to a table it is a relationship; if it resolves to nothing it is a namespace.
*/
var parentNode = dagBuilder.getNode(identifier);
checkFatal(
parentNode.isPresent(),
sqrlDef.getTableName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Could not find parent table for relationship: %s",
tblFnStmt.getPath().getFirst());
checkFatal(
parentNode.get() instanceof TableNode,
sqrlDef.getTableName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Relationships can only be added to tables (not functions): %s [%s]",
tblFnStmt.getPath().getFirst(),
parentNode.get().getClass());
identifier = scriptContext.toIdentifier(tablePath.toString());
parentTbl = ((TableNode) parentNode.get()).getTableAnalysis();
checkFatal(
parentTbl.getOptionalBaseTable().isEmpty(),
ErrorCode.BASETABLE_ONLY_ERROR,
"Relationships can only be added to the base table [%s]",
parentTbl.getBaseTable().getIdentifier());
if (parentNode.isEmpty()) {
namespaced = true;
namespaceDef = namespaces.get(namespaceName);
checkFatal(
tblFnStmt.getArgumentsByIndex().stream().noneMatch(ParsedArgument::isParentField),
sqrlDef.getTableName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Namespaced function [%s] cannot reference `this`. It has no parent table.",
tblFnStmt.getPath());
log.info(
"Exposing [{}] under GraphQL namespace [{}]",
tblFnStmt.getPath(),
tblFnStmt.getPath().getFirst());
} else {
checkFatal(
parentNode.get() instanceof TableNode,
sqrlDef.getTableName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Relationships can only be added to tables (not functions): %s [%s]",
tblFnStmt.getPath().getFirst(),
parentNode.get().getClass());
parentTbl = ((TableNode) parentNode.get()).getTableAnalysis();
checkFatal(
parentTbl.getOptionalBaseTable().isEmpty(),
ErrorCode.BASETABLE_ONLY_ERROR,
"Relationships can only be added to the base table [%s]",
parentTbl.getBaseTable().getIdentifier());
}
}
// Resolve arguments, map indexes, and check for errors
Map<Integer, Integer> argumentIndexMap = new HashMap<>();
Expand All @@ -455,7 +480,38 @@ private void planStatement(
fieldName, argIndex.withResolvedType(field.getType(), arguments.size()));
}
}
var signatureArg = arguments.get(Name.system(argIndex.getName().get()));
var argName = argIndex.getName().get();
// References to a namespace parameter use a `<namespace>.` prefix (e.g.
// `:admin.asTenantId`).
// External namespace params are bound as parent-fields from the namespace field; claims
// as
// metadata. Both are inherited automatically from the CREATE NAMESPACE declaration.
var namespaceParam = namespaceParamName(argName, namespaceName);
Name lookupKey;
if (namespaceParam.isPresent()) {
var paramName = namespaceParam.get();
checkFatal(
namespaceDef != null,
argIndex.getName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Namespace [%s] is not declared. Declare it with CREATE NAMESPACE.",
namespaceName);
var param = namespaceDef.getParam(paramName);
checkFatal(
param.isPresent(),
argIndex.getName().getFileLocation(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Namespace [%s] has no parameter [%s]",
namespaceName,
paramName);
lookupKey = Name.system(paramName);
if (!arguments.containsKey(lookupKey)) {
arguments.put(lookupKey, namespaceParamToArgument(param.get(), arguments.size()));
}
} else {
lookupKey = Name.system(argName);
}
var signatureArg = arguments.get(lookupKey);
checkFatal(
signatureArg != null,
argIndex.getName().getFileLocation(),
Expand Down Expand Up @@ -513,6 +569,10 @@ private void planStatement(
errors);
}
fnBuilder.fullPath(tblFnStmt.getPath());
fnBuilder.namespaced(namespaced);
if (namespaceDef != null) {
fnBuilder.namespaceArguments(namespaceFieldArguments(namespaceDef));
}
var visibility =
new AccessVisibility(
access, hints.isTest(), tblFnStmt.isRelationship() || passthroughFn, isHidden);
Expand Down Expand Up @@ -596,6 +656,84 @@ private boolean shouldExcludeTestTable(PlannerHints hints) {
return hints.isTest() && executionGoal != ExecutionGoal.TEST;
}

/**
* Validates and registers a {@code CREATE NAMESPACE} declaration. Each parameter is either an
* external argument (exposed on the namespace field) or a hidden metadata claim (with {@code
* METADATA FROM}).
*/
private void addNamespace(
SqrlCreateNamespaceStatement nsStmt, Sqrl2FlinkSQLTranslator sqrlEnv, ErrorCollector errors) {
var name = nsStmt.getName().get();
errors.checkFatal(
!namespaces.containsKey(name),
ErrorCode.INVALID_SQRL_DEFINITION,
"Namespace [%s] is already defined",
name);
List<NamespaceDefinition.Param> params = new ArrayList<>();
for (var parsedField : sqrlEnv.parse2RelDataType(nsStmt.getParams())) {
var field = parsedField.field();
var metadata =
parsedField
.metadata()
.map(
metaStr -> {
var resolved =
SqrlTableFunctionStatement.parseMetadata(
metaStr, !field.getType().isNullable());
errors.checkFatal(
resolved.isPresent(),
ErrorCode.INVALID_TABLE_FUNCTION_ARGUMENTS,
"Invalid metadata key provided: %s",
metaStr);
return resolved.get();
});
params.add(new NamespaceDefinition.Param(field.getName(), field.getType(), metadata));
}
namespaces.put(name, new NamespaceDefinition(name, params));
}

/**
* If {@code argName} is a reference to a parameter of the given namespace (i.e. {@code
* <namespace>.<param>}), returns the bare parameter name.
*/
private static Optional<String> namespaceParamName(String argName, Name namespaceName) {
if (namespaceName == null) {
return Optional.empty();
}
var prefix = namespaceName.getDisplay() + ".";
if (argName.regionMatches(true, 0, prefix, 0, prefix.length())) {
return Optional.of(argName.substring(prefix.length()));
}
return Optional.empty();
}

/**
* Builds the function parameter injected into a namespaced function for a namespace parameter: an
* external parameter is a parent-field (bound from the namespace field's argument), a claim is
* metadata (bound from the JWT).
*/
private static ParsedArgument namespaceParamToArgument(
NamespaceDefinition.Param param, int index) {
return new ParsedArgument(
new ParsedObject<>(param.name(), FileLocation.START),
param.type(),
param.metadata(),
Optional.empty(),
param.isExternal(),
index);
}

/** The external namespace parameters exposed as arguments on the namespace field. */
private static List<FunctionParameter> namespaceFieldArguments(NamespaceDefinition namespaceDef) {
var external = namespaceDef.externalParams();
List<FunctionParameter> args = new ArrayList<>();
for (var i = 0; i < external.size(); i++) {
var param = external.get(i);
args.add(new SqrlFunctionParameter(param.name(), i, param.type()));
}
return args;
}

/**
* Adjusts the access for functions and tables based on the available stages and configuration We
* might consider throwing an exception for SUBSCRIPTION access when no subscription stages are
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright © 2021 DataSQRL (contact@datasqrl.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.datasqrl.planner.parser;

import com.datasqrl.canonicalizer.Name;
import com.datasqrl.error.ErrorLocation.FileLocation;
import lombok.Value;

/**
* Represents a {@code CREATE NAMESPACE <name> ( <params> );} statement that groups queries and
* mutations under a GraphQL sub-object (e.g. {@code backend { ... }}) and declares parameters
* shared by every function in the namespace. A parameter with {@code METADATA FROM 'auth....'} is a
* hidden JWT claim; a parameter without it becomes an argument exposed on the namespace field
* itself. Functions join the namespace via the {@code <name>.func} path prefix and reference the
* shared parameters as {@code :<name>.<param>}.
*/
@Value
public class SqrlCreateNamespaceStatement implements SqrlStatement {

ParsedObject<Name> name;
ParsedObject<String> params;
SqrlComments comments;

@Override
public FileLocation getDefaultLocation() {
return name.getFileLocation();
}
}
Loading