From 425e44f4aa9e80193e7a3ef094f33b4278dee63e Mon Sep 17 00:00:00 2001 From: anderson-joyle Date: Fri, 19 Jan 2024 16:46:12 -0600 Subject: [PATCH] Collect function. --- .../Entities/External/IExternalDataSource.cs | 2 + .../Localization/Strings.cs | 4 + .../Texl/Builtins/Collect.cs | 374 ++++++++++++++++++ .../Microsoft.PowerFx.Core/Types/DType.cs | 2 + .../Utils/MutationUtils.cs | 67 ++++ .../Environment/PowerFxConfigExtensions.cs | 4 +- .../Functions/LibraryMutation.cs | 134 +++++++ .../Functions/Mutation/MutationUtils.cs | 35 -- src/strings/PowerFxResources.en-US.resx | 10 + .../TestDVEntity.cs | 2 + .../ExpressionTestCases/Collect.txt | 18 +- .../ExpressionTestCases/Collect_V1Compat.txt | 2 +- .../Collect_V1CompatDisabled.txt | 2 +- .../Helpers/TestTabularDataSource.cs | 2 + .../CollectFunctionTests.cs | 2 +- .../MutationFunctionsTests.cs | 20 + .../MutationScripts/Collect.txt | 30 +- 17 files changed, 664 insertions(+), 46 deletions(-) create mode 100644 src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs create mode 100644 src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs create mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs delete mode 100644 src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs diff --git a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs index ed7791bb8a..e5e184e182 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalDataSource.cs @@ -19,6 +19,8 @@ internal interface IExternalDataSource : IExternalEntity, IExternalPageableSymbo bool RequiresAsync { get; } + bool IsWritable { get; } + IExternalDataEntityMetadataProvider DataEntityMetadataProvider { get; } DataSourceKind Kind { get; } diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 79dcea3ffa..a23ee247a3 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -485,6 +485,10 @@ internal static class TexlStrings public static StringGetter CollectDataSourceArg = (b) => StringResources.Get("CollectDataSourceArg", b); public static StringGetter CollectRecordArg = (b) => StringResources.Get("CollectRecordArg", b); + public static StringGetter CollectItemArg = (b) => StringResources.Get("CollectItemArg", b); + public static StringGetter AboutCollect_data_source = (b) => StringResources.Get("AboutCollect_data_source", b); + public static StringGetter AboutCollect_item = (b) => StringResources.Get("AboutCollect_item", b); + public static StringGetter AboutClearCollect = (b) => StringResources.Get("AboutClearCollect", b); public static StringGetter ClearCollectDataSourceArg = (b) => StringResources.Get("ClearCollectDataSourceArg", b); public static StringGetter ClearCollectRecordArg = (b) => StringResources.Get("ClearCollectRecordArg", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs new file mode 100644 index 0000000000..ce24b5fa11 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Collect.cs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PowerFx.Core.App.ErrorContainers; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Entities; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Functions.FunctionArgValidators; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Core.Utils; +using Microsoft.PowerFx.Syntax; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Core.Texl.Builtins +{ + // Collect(collection:*[...], item1:![...]|*[...], ...) + internal class CollectFunction : BuiltinFunction + { + public override bool AffectsCollectionSchemas => true; + + public bool CanSuggestThisItem => true; + + public override bool SupportsParamCoercion => false; + + public override bool ManipulatesCollections => true; + + public override bool ModifiesValues => true; + + public override bool IsSelfContained => false; + + public override bool RequiresDataSourceScope => true; + + protected virtual bool IsScalar => false; + + public override bool CanSuggestInputColumns => true; + + public override bool MutatesArg0 => true; + + /// + /// Since Arg1 and Arg2 depends on type of Arg1 return false for them. + /// + public override bool TryGetTypeForArgSuggestionAt(int argIndex, out DType type) + { + if (argIndex == 1 || argIndex == 2) + { + type = default; + return false; + } + + return base.TryGetTypeForArgSuggestionAt(argIndex, out type); + } + + public override bool ArgMatchesDatasourceType(int argNum) + { + return argNum >= 1; + } + + public override bool IsLazyEvalParam(int index, Features features) + { + // First argument to mutation functions is Lazy for datasources that are copy-on-write. + // If there are any side effects in the arguments, we want those to have taken place before we make the copy. + return index == 0; + } + + public CollectFunction() + : this("Collect", TexlStrings.AboutCollect) + { + } + + protected CollectFunction(string name, TexlStrings.StringGetter description) + : base(name, description, FunctionCategories.Behavior, DType.EmptyRecord, 0, 2, int.MaxValue, DType.EmptyTable) + { + } + + public override IEnumerable GetSignatures() + { + yield return new[] { TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg }; + } + + public override IEnumerable GetSignatures(int arity) + { + if (arity > 2) + { + return GetGenericSignatures(arity, TexlStrings.CollectDataSourceArg, TexlStrings.CollectItemArg); + } + + return base.GetSignatures(arity); + } + + public virtual DType GetCollectedType(PowerFx.Features features, DType argType) + { + Contracts.Assert(argType.IsValid); + + return argType; + } + + // Attempt to get the unified schema of the items being collected by an invocation. + private bool TryGetUnifiedCollectedType(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType collectedType) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + var fValid = true; + + DType itemType = DType.Invalid; + DType datasourceType = argTypes[0]; + + var argc = args.Length; + + for (var i = 1; i < argc; i++) + { + DType argType = GetCollectedType(context.Features, argTypes[i]); + + // The subsequent args should all be aggregates. + if (!argType.IsAggregate) + { + errors.EnsureError(args[i], TexlStrings.ErrBadType_Type, argType.GetKindString()); + fValid = false; + continue; + } + + // Promote the arg type to a table to facilitate unioning. + if (!argType.IsRecord) + { + argType = argType.ToRecord(); + } + + // Checks if all record names exist against table type and if its possible to coerce. + bool checkAggregateNames = argType.CheckAggregateNames(datasourceType, args[i], errors, context.Features, SupportsParamCoercion); + fValid = fValid && checkAggregateNames; + + if (!itemType.IsValid) + { + itemType = argType; + } + else + { + var fUnionError = false; + itemType = DType.Union(ref fUnionError, itemType, argType, useLegacyDateTimeAccepts: true, context.Features); + if (fUnionError) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrIncompatibleTypes); + fValid = false; + } + } + + // We only support accessing entities in collections if the collection has only 1 argument that contributes to it's type + if (argc != 2 && itemType.ContainsDataEntityType(DPath.Root)) + { + fValid &= DropAllOfKindNested(ref itemType, errors, args[i], DKind.DataEntity); + } + } + + Contracts.Assert(!itemType.IsValid || itemType.IsRecord); + collectedType = itemType.IsValid ? itemType : DType.EmptyRecord; + return fValid; + } + + // Typecheck an invocation of Collect. + public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DType[] argTypes, IErrorContainer errors, out DType returnType, out Dictionary nodeToCoercedTypeMap) + { + Contracts.AssertValue(args); + Contracts.AssertAllValues(args); + Contracts.AssertValue(argTypes); + Contracts.Assert(args.Length == argTypes.Length); + Contracts.AssertValue(errors); + Contracts.Assert(MinArity <= args.Length && args.Length <= MaxArity); + + var fValid = base.CheckTypes(context, args, argTypes, errors, out returnType, out nodeToCoercedTypeMap); + + // Need a collection for the 1st arg + DType collectionType = argTypes[0]; + if (!collectionType.IsTable) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrInvalidArgs_Func, Name); + fValid = false; + } + + // Get the unified collected type on the RHS. This will generate appropriate + // document errors for invalid arguments such as unsupported aggregate types. + fValid &= TryGetUnifiedCollectedType(context, args, argTypes, errors, out DType collectedType); + Contracts.Assert(collectedType.IsRecord); + + if (fValid) + { + if (!collectedType.TryGetCoercionSubType(collectionType, out DType coercionType, out var coercionNeeded, context.Features)) + { + fValid = false; + } + else + { + if (coercionNeeded) + { + CollectionUtils.Add(ref nodeToCoercedTypeMap, args[1], coercionType); + } + + var fError = false; + + returnType = DType.Union(ref fError, collectionType.ToRecord(), collectedType, useLegacyDateTimeAccepts: false, context.Features, allowCoerce: true); + + if (argTypes.Length == 2) + { + if (argTypes[1].IsTable && argTypes[1].Kind != DKind.ObjNull) + { + returnType = returnType.ToTable(); + } + } + else + { + returnType = returnType.ToTable(); + } + + if (fError) + { + fValid = false; + if (!SetErrorForMismatchedColumns(collectionType, collectedType, args[1], errors, context.Features)) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[0], TexlStrings.ErrTableDoesNotAcceptThisType); + } + } + } + } + + return fValid; + } + + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + DType dataSourceType = argTypes[0]; + bool isConnected = binding.EntityScope != null + && binding.EntityScope.TryGetDataSource(args[0], out IExternalDataSource dataSourceInfo) + && (dataSourceInfo.Kind == DataSourceKind.Connected || dataSourceInfo.Kind == DataSourceKind.CdsNative); + + if (isConnected) + { + for (int i = 1; i < args.Length; i++) + { + DType curType = argTypes[i]; + foreach (var typedName in curType.GetNames(DPath.Root)) + { + DName name = typedName.Name; + if (!dataSourceType.TryGetType(name, out DType dsNameType)) + { + dataSourceType.ReportNonExistingName(FieldNameKind.Display, errors, name, args[i], DocumentErrorSeverity.Warning); + } + } + } + } + + base.CheckSemantics(binding, args, argTypes, errors); + base.ValidateArgumentIsMutable(binding, args[0], errors); + + int skip = 1; + + MutationUtils.CheckSemantics(binding, this, args, argTypes, errors); + MutationUtils.CheckForReadOnlyFields(argTypes[0], args.Skip(skip).ToArray(), argTypes.Skip(skip).ToArray(), errors); + } + + // This method returns true if there are special suggestions for a particular parameter of the function. + public override bool HasSuggestionsForParam(int argumentIndex) + { + Contracts.Assert(argumentIndex >= 0); + + return argumentIndex == 0; + } + + public override bool TryGetDataSourceNodes(CallNode callNode, TexlBinding binding, out IList dsNodes) + { + Contracts.AssertValue(callNode); + Contracts.AssertValue(binding); + + dsNodes = new List(); + if (callNode.Args.Count != 2) + { + return false; + } + + var args = Contracts.VerifyValue(callNode.Args.Children); + var arg1 = Contracts.VerifyValue(args[1]); + + // Only the second arg can contribute to the output for the purpose of delegation + return ArgValidators.DataSourceArgNodeValidator.TryGetValidValue(arg1, binding, out dsNodes); + } + + public override IEnumerable GetIdentifierOfModifiedValue(TexlNode[] args, out TexlNode identifierNode) + { + Contracts.AssertValue(args); + + identifierNode = null; + if (args.Length == 0) + { + return null; + } + + var firstNameNode = args[0]?.AsFirstName(); + identifierNode = firstNameNode; + + if (firstNameNode == null) + { + return null; + } + + var identifiers = new List + { + firstNameNode.Ident + }; + return identifiers; + } + + public override bool IsAsyncInvocation(CallNode callNode, TexlBinding binding) + { + Contracts.AssertValue(callNode); + Contracts.AssertValue(binding); + + return Arg0RequiresAsync(callNode, binding); + } + + public static DType GetCollectedTypeForGivenArgType(Features features, DType argType) + { + Contracts.Assert(argType.IsValid); + + if (!argType.IsPrimitive) + { + return argType; + } + + // Passed a scalar; make a record out of it, using a name that depends on the type. + string fieldName = Contracts.VerifyValue(CreateInvariantFieldName(features, argType.Kind)); + return DType.CreateRecord(new TypedName[] { new TypedName(argType, new DName(fieldName)) }); + } + + protected static string CreateInvariantFieldName(PowerFx.Features features, DKind dKind) + { + Contracts.Assert(dKind >= DKind._Min && dKind < DKind._Lim); + + return GetScalarSingleColumnNameForType(features, dKind); + } + + private static string GetScalarSingleColumnNameForType(Features features, DKind kind) + { + return kind switch + { + DKind.Image or + DKind.Hyperlink or + DKind.Media or + DKind.Blob or + DKind.PenImage => features.ConsistentOneColumnTableResult ? TableValue.ValueName : "Url", + + _ => TableValue.ValueName + }; + } + } + + // Collect(collection:*[...], item1, ...) + internal class CollectScalarFunction : CollectFunction + { + protected override bool IsScalar => true; + + public override bool SupportsParamCoercion => false; + + public override DType GetCollectedType(PowerFx.Features features, DType argType) + { + return GetCollectedTypeForGivenArgType(features, argType); + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs index a4ef4f77ff..d37679f0f6 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/DType.cs @@ -637,6 +637,8 @@ public bool IsActivityPointer public bool HasPolymorphicInfo => PolymorphicInfo != null; + public bool IsSingleColumnTable => IsTable && GetNames(DPath.Root).Count() == 1; + /// /// Whether this type is a subtype of all possible types, meaning that it can be placed in /// any location without coercion. diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs new file mode 100644 index 0000000000..cf5d4fe3c4 --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Core/Utils/MutationUtils.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Linq; +using Microsoft.PowerFx.Core.App.ErrorContainers; +using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Entities; +using Microsoft.PowerFx.Core.Errors; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Localization; +using Microsoft.PowerFx.Core.Types; +using Microsoft.PowerFx.Syntax; + +namespace Microsoft.PowerFx.Core.Utils +{ + internal class MutationUtils + { + public static void CheckForReadOnlyFields(DType dataSourceType, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + if (dataSourceType.AssociatedDataSources.Any()) + { + var tableDsInfo = dataSourceType.AssociatedDataSources.Single(); + + if (tableDsInfo is IExternalCdsDataSource cdsTableInfo) + { + for (int i = 0; i < argTypes.Length; i++) + { + if (!cdsTableInfo.IsArgTypeValidForMutation(argTypes[i], out var invalidFieldNames)) + { + errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrRecordContainsInvalidFields_Arg, string.Join(", ", invalidFieldNames)); + } + } + } + } + } + + /// + /// Adds specific errors for mutation functions. + /// + public static void CheckSemantics(TexlBinding binding, TexlFunction function, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + var targetArg = args[0]; + + // control!property are not valid targets for Collect, Remove, etc. + DottedNameNode dotted; + if ((dotted = targetArg.AsDottedName()) != null + && binding.TryCastToFirstName(dotted.Left, out var firstNameInfo) + && firstNameInfo.Kind == BindKind.Control) + { + errors.EnsureError(targetArg, TexlStrings.ErrInvalidArgs_Func, function.Name); + return; + } + + // Checks for something similar to Collect(x.a, 4). + if (!binding.TryCastToFirstName(targetArg, out firstNameInfo)) + { + return; + } + + if (firstNameInfo.Data is IExternalDataSource info && !info.IsWritable) + { + errors.EnsureError(targetArg, TexlStrings.ErrInvalidArgs_Func, function.Name); + return; + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs index 8ea79c6a0d..c024274451 100644 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Environment/PowerFxConfigExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.Texl.Builtins; using Microsoft.PowerFx.Functions; using Microsoft.PowerFx.Interpreter; @@ -37,11 +38,12 @@ public static void EnableSetFunction(this PowerFxConfig powerFxConfig) public static void EnableMutationFunctions(this SymbolTable symbolTable) { symbolTable.AddFunction(new RecalcEngineSetFunction()); - symbolTable.AddFunction(new CollectFunction()); symbolTable.AddFunction(new PatchFunction()); symbolTable.AddFunction(new RemoveFunction()); symbolTable.AddFunction(new ClearFunction()); symbolTable.AddFunction(new ClearCollectFunction()); + symbolTable.AddFunction(new CollectImpl()); + symbolTable.AddFunction(new CollectScalarImpl()); } [Obsolete("RegEx is still in preview. Grammar may change.")] diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs new file mode 100644 index 0000000000..cc8c3a392e --- /dev/null +++ b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryMutation.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PowerFx.Core.Functions; +using Microsoft.PowerFx.Core.IR; +using Microsoft.PowerFx.Functions; +using Microsoft.PowerFx.Interpreter; +using Microsoft.PowerFx.Types; + +namespace Microsoft.PowerFx.Core.Texl.Builtins +{ + internal class CollectImpl : CollectFunction, IAsyncTexlFunction + { + public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + { + return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + } + } + + internal class CollectScalarImpl : CollectScalarFunction, IAsyncTexlFunction + { + public async Task InvokeAsync(FormulaValue[] args, CancellationToken cancellationToken) + { + return await new CollectProcess().Process(args, cancellationToken).ConfigureAwait(false); + } + } + + internal class CollectProcess + { + internal async Task Process(FormulaValue[] args, CancellationToken cancellationToken) + { + FormulaValue arg0; + var argc = args.Length; + var returnIsTable = args.Length > 2; + + // Need to check if the Lazy first argument has been evaluated since it may have already been + // evaluated in the ClearCollect case. + if (args[0] is LambdaFormulaValue arg0lazy) + { + arg0 = await arg0lazy.EvalAsync().ConfigureAwait(false); + } + else + { + arg0 = args[0]; + } + + if (arg0 is BlankValue) + { + return arg0; + } + + if (arg0 is ErrorValue) + { + return arg0; + } + + if (arg0 is not TableValue) + { + return CommonErrors.RuntimeTypeMismatch(IRContext.NotInSource(arg0.Type)); + } + + var tableValue = arg0 as TableValue; + + List> resultRows = new List>(); + + for (int i = 1; i < argc; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var arg = args[i]; + + if (arg is TableValue argTableValue) + { + returnIsTable = true; + + foreach (DValue row in argTableValue.Rows) + { + if (row.IsBlank) + { + continue; + } + else if (row.IsError) + { + return row.Error; + } + else + { + var recordValueCopy = (RecordValue)row.ToFormulaValue().MaybeShallowCopy(); + resultRows.Add(await tableValue.AppendAsync(recordValueCopy, cancellationToken).ConfigureAwait(false)); + } + } + } + else if (arg is RecordValue) + { + var recordValueCopy = (RecordValue)arg.MaybeShallowCopy(); + resultRows.Add(await tableValue.AppendAsync(recordValueCopy, cancellationToken).ConfigureAwait(false)); + } + else if (arg is ErrorValue) + { + return arg; + } + else if (arg is BlankValue && !tableValue.Type._type.IsSingleColumnTable) + { + continue; + } + else + { + // If arg is a scalar value, then we need to create a single column record. + NamedValue namedValue = new NamedValue(tableValue.Type.SingleColumnFieldName, arg); + var singleColumnRecord = FormulaValue.NewRecordFromFields(namedValue); + + resultRows.Add(await tableValue.AppendAsync(singleColumnRecord, cancellationToken).ConfigureAwait(false)); + } + } + + if (resultRows.Count == 0) + { + return FormulaValue.NewBlank(arg0.Type); + } + else if (returnIsTable) + { + return CompileTimeTypeWrapperTableValue.AdjustType(tableValue.Type, new InMemoryTableValue(IRContext.NotInSource(arg0.Type), resultRows)); + } + else + { + return CompileTimeTypeWrapperRecordValue.AdjustType(tableValue.Type.ToRecord(), (RecordValue)resultRows.First().ToFormulaValue()); + } + } + } +} diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs b/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs deleted file mode 100644 index 224c9d1cd7..0000000000 --- a/src/libraries/Microsoft.PowerFx.Interpreter/Functions/Mutation/MutationUtils.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Linq; -using Microsoft.PowerFx.Core.App.ErrorContainers; -using Microsoft.PowerFx.Core.Entities; -using Microsoft.PowerFx.Core.Errors; -using Microsoft.PowerFx.Core.Localization; -using Microsoft.PowerFx.Core.Types; -using Microsoft.PowerFx.Syntax; - -namespace Microsoft.PowerFx.Interpreter -{ - internal class MutationUtils - { - public static void CheckForReadOnlyFields(DType dataSourceType, TexlNode[] args, DType[] argTypes, IErrorContainer errors) - { - if (dataSourceType.AssociatedDataSources.Any()) - { - var tableDsInfo = dataSourceType.AssociatedDataSources.Single(); - - if (tableDsInfo is IExternalCdsDataSource cdsTableInfo) - { - for (int i = 0; i < argTypes.Length; i++) - { - if (!cdsTableInfo.IsArgTypeValidForMutation(argTypes[i], out var invalidFieldNames)) - { - errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrRecordContainsInvalidFields_Arg, string.Join(", ", invalidFieldNames)); - } - } - } - } - } - } -} diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 3a1f4e8ff8..4a51b2de7c 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4426,4 +4426,14 @@ **Disclaimer:** AI-generated content can have mistakes. Make sure it's accurate and appropriate before using it. [See terms](https://go.microsoft.com/fwlink/?linkid=2225491) The disclaimer we show on AI functions. This is github flavored markdown. So ** marks text in bold. + + The data source that you want to add data to. + + + A record or table to collect. A record will be appended to the datasource. A table will have its rows appended to the datasource. + + + item + function_parameter - Second parameter for the Collect function. The item to be added. + \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs index e93b903077..f0bda8e028 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests/AssociatedDataSourcesTests/TestDVEntity.cs @@ -30,6 +30,8 @@ public class AccountsEntity : IExternalEntity, IExternalDataSource public bool IsPageable => true; + public bool IsWritable => true; + DType IExternalEntity.Type => AccountsTypeHelper.GetDType(); IExternalDataEntityMetadataProvider IExternalDataSource.DataEntityMetadataProvider => throw new NotImplementedException(); diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt index 80bcf18900..8e9b0c6959 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect.txt @@ -27,19 +27,19 @@ true {Field1:3,Field2:"mars",Field3:DateTime(2022,3,1,0,0,0,0),Field4:false} >> Collect(t1) -Errors: Error 0-11: Invalid number of arguments: received 1, expected 2. +Errors: Error 0-11: Invalid number of arguments: received 1, expected 2 or more. >> Collect(t1, r2, r2) -Errors: Error 0-19: Invalid number of arguments: received 3, expected 2. +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) >> Collect(t1, r2, 1; 2; r2) -Errors: Error 0-25: Invalid number of arguments: received 3, expected 2. +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) >> Collect(t1, "x") -Errors: Error 12-15: Invalid argument type (Text). Expecting a Record value instead.|Error 12-15: Invalid argument type. Cannot use Text values in this context.|Error 0-16: The function 'Collect' has some invalid arguments. +Errors: Error 0-16: The function 'Collect' has some invalid arguments.|Error 12-15: Invalid argument type. Cannot use Text values in this context. >> Collect(t1, 1) -Errors: Error 12-13: Invalid argument type (Decimal). Expecting a Record value instead.|Error 12-13: Invalid argument type. Cannot use Decimal values in this context.|Error 0-14: The function 'Collect' has some invalid arguments. +Errors: Error 0-14: The function 'Collect' has some invalid arguments.|Error 12-13: Invalid argument type. Cannot use Decimal values in this context. >> Collect(Foo,r2) Errors: Error 8-11: Name isn't valid. 'Foo' isn't recognized.|Error 12-14: The specified column 'Field1' does not exist.|Error 0-15: The function 'Collect' has some invalid arguments. @@ -54,4 +54,10 @@ Errors: Error 12-15: Name isn't valid. 'Foo' isn't recognized. Errors: Error 11-22: The specified column 'Price' does not exist.|Error 0-23: The function 'Collect' has some invalid arguments.|Error 23-29: Name isn't valid. 'Price' isn't recognized. >> Collect(t_empty,{Value:200}).Value -200 \ No newline at end of file +200 + +>> Collect(t1, Table(r2,r2)) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) + +>> Collect(t1, Table(r2,r2), {Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) +Table({Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false},{Field1:2,Field2:"moon",Field3:DateTime(2022,2,1,0,0,0,0),Field4:false}) \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt index db0ffc0398..dbc64f52b6 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1Compat.txt @@ -19,4 +19,4 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 8-15: T Errors: Error 8-15: The value passed to the 'Collect' function cannot be changed. >> Collect("", "") -Errors: Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type (Text). Expecting a Record value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed.|Error 0-15: The function 'Collect' has some invalid arguments. +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 8-10: The value passed to the 'Collect' function cannot be changed. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt index 228b45371b..39237bfccb 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Collect_V1CompatDisabled.txt @@ -19,4 +19,4 @@ Errors: Error 17-19: The specified column 'Field1' does not exist.|Error 0-20: T Blank() >> Collect("", "") -Errors: Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type (Text). Expecting a Record value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context.|Error 0-15: The function 'Collect' has some invalid arguments. +Errors: Error 0-15: The function 'Collect' has some invalid arguments.|Error 8-10: Invalid argument type (Text). Expecting a Table value instead.|Error 12-14: Invalid argument type. Cannot use Text values in this context. \ No newline at end of file diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs b/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs index cb708b3a17..5b2e98b43a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests/Helpers/TestTabularDataSource.cs @@ -219,6 +219,8 @@ internal TestDataSource(string name, DType schema, string[] keyColumns = null, I IDelegationMetadata IExternalDataSource.DelegationMetadata => DelegationMetadata; + public bool IsWritable => throw new NotImplementedException(); + public bool CanIncludeExpand(IExpandInfo expandToAdd) { throw new NotImplementedException(); diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs index c1673211d7..4f0fafd52b 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/CollectFunctionTests.cs @@ -61,7 +61,7 @@ public async Task AppendCountTest(string script, int expected) [InlineData("Collect(lazyTable, lazyCoercibleRecord)", true)] [InlineData("Collect(lazyTable, lazyNotCoercibleRecord)", false)] [InlineData("Collect(lazyTable, {Value:1})", false)] - [InlineData("Collect(lazyTable, lazyTable)", false)] + [InlineData("Collect(lazyTable, lazyTable)", true)] [InlineData("Collect(lazyRecord, lazyRecord)", false)] [InlineData("Collect(lazyRecord, lazyTable)", false)] diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs index ce59e88dbb..70a16670ee 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationFunctionsTests.cs @@ -364,6 +364,26 @@ public void DontCopyDerivedRecordValuesTest(string expression) Assert.Equal("x", fileObjectRecordValue.SomeProperty); } + [Fact] + public void SymbolTableEnableMutationFuntionsTest() + { + var expr = "Collect()"; + var engine = new RecalcEngine(); + + var symbolTable = new SymbolTable(); + var symbolTableEnabled = new SymbolTable(); + + symbolTableEnabled.EnableMutationFunctions(); + + // Mutation functions not listed. + var check = engine.Check(expr, symbolTable: symbolTable); + Assert.DoesNotContain(check.Symbols.Functions.FunctionNames, f => f == "Collect"); + + // Mutation functions is listed. + var checkEnabled = engine.Check(expr, symbolTable: symbolTableEnabled); + Assert.Contains(checkEnabled.Symbols.Functions.FunctionNames, f => f == "Collect"); + } + internal class FileObjectRecordValue : InMemoryRecordValue { public string SomeProperty { get; set; } diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt index c76b69f83a..98c150f492 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/MutationScripts/Collect.txt @@ -49,7 +49,7 @@ Blank() Table({Value:1},{Value:2},{Value:3}) >> Collect(temp1,{Value:"200"}).Value -Errors: Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-28: The function 'Collect' has some invalid arguments.|Error 28-34: Name isn't valid. 'Value' isn't recognized. +Errors: Error 14-27: The type of this argument 'Value' does not match the expected type 'Decimal'. Found type 'Text'.|Error 0-28: The function 'Collect' has some invalid arguments. >> Collect( temp1, { Value:"11"+0 } ) {Value:11} @@ -86,3 +86,31 @@ Table({Value:1},{Value:2},{Value:3},{Value:4}) >> With({r:{Value:5}}, Collect(t, r); Patch(t, Last(t), {Value:-1}); r) {Value:5} + +>> Set(t1, [1,2,3]) +Table({Value:1},{Value:2},{Value:3}) + +>> Collect(t1, 4) +{Value:4} + +>> Collect(t1, [5,6,7],8,9,10) +Table({Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) + +>> 0;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10}) + +>> Collect(t1, Table({Value:11},Blank())) +Table({Value:11}) + +>> Collect(t1, Table({Value:12},Error({Kind:ErrorKind.Custom}))) +Error({Kind:ErrorKind.Custom}) + +>> Collect(t1, [13,Blank(),14]) +Table({Value:13},{Value:Blank()},{Value:14}) + +// {Value:16} wont be collected because it appears after the error. +>> Collect(t1, Table({Value:15}, Error({Kind:ErrorKind.Custom}), {Value:16})) +Error({Kind:ErrorKind.Custom}) + +>> 1;t1 +Table({Value:1},{Value:2},{Value:3},{Value:4},{Value:5},{Value:6},{Value:7},{Value:8},{Value:9},{Value:10},{Value:11},{Value:12},{Value:13},{Value:Blank()},{Value:14},{Value:15}) \ No newline at end of file